diff --git a/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index b95fa733a..e061037db 100644 --- a/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -9,6 +9,20 @@ import android.util.Log import android.view.accessibility.AccessibilityEvent import androidx.annotation.Keep import androidx.annotation.RequiresApi +import java.util.* + +const val LIFT_DOWN = 9 +const val LIFT_MOVE = 8 +const val LIFT_UP = 10 +const val RIGHT_UP = 18 +const val WHEEL_BUTTON_DOWN = 33 +const val WHEEL_BUTTON_UP = 34 +const val WHEEL_DOWN = 523331 +const val WHEEL_UP = 963 + +const val WHEEL_STEP = 120 +const val WHEEL_DURATION = 50L +const val LONG_TAP_DELAY = 200L class InputService : AccessibilityService() { @@ -26,10 +40,15 @@ class InputService : AccessibilityService() { private val logTag = "input service" private var leftIsDown = false - private var mPath = Path() - private var mLastGestureStartTime = 0L + private var touchPath = Path() + private var lastTouchGestureStartTime = 0L private var mouseX = 0 private var mouseY = 0 + private var timer = Timer() + private var recentActionTask: TimerTask? = null + + private val wheelActionsQueue = LinkedList() + private var isWheelActionsPolling = false @Keep @RequiresApi(Build.VERSION_CODES.N) @@ -46,15 +65,16 @@ class InputService : AccessibilityService() { _y } - if (!(mask == 9 || mask == 10)) { - mouseX = x * INFO.scale - mouseY = y * INFO.scale + if (mask == 0 || mask == LIFT_MOVE) { + mouseX = x * SCREEN_INFO.scale + mouseY = y * SCREEN_INFO.scale } // left button down ,was up - if (mask == 9) { + if (mask == LIFT_DOWN) { leftIsDown = true startGesture(mouseX, mouseY) + return } // left down ,was down @@ -63,32 +83,118 @@ class InputService : AccessibilityService() { } // left up ,was down - if (mask == 10) { + if (mask == LIFT_UP) { leftIsDown = false endGesture(mouseX, mouseY) + return + } + + if (mask == RIGHT_UP) { + performGlobalAction(GLOBAL_ACTION_BACK) + return + } + + // long WHEEL_BUTTON_DOWN -> GLOBAL_ACTION_RECENTS + if (mask == WHEEL_BUTTON_DOWN) { + timer.purge() + recentActionTask = object : TimerTask() { + override fun run() { + performGlobalAction(GLOBAL_ACTION_RECENTS) + recentActionTask = null + } + } + timer.schedule(recentActionTask, LONG_TAP_DELAY) + } + + // wheel button up + if (mask == WHEEL_BUTTON_UP) { + if (recentActionTask != null) { + recentActionTask!!.cancel() + performGlobalAction(GLOBAL_ACTION_HOME) + } + return + } + + if (mask == WHEEL_DOWN) { + if (mouseY < WHEEL_STEP) { + return + } + val path = Path() + path.moveTo(mouseX.toFloat(), mouseY.toFloat()) + path.lineTo(mouseX.toFloat(), (mouseY - WHEEL_STEP).toFloat()) + val stroke = GestureDescription.StrokeDescription( + path, + 0, + WHEEL_DURATION + ) + val builder = GestureDescription.Builder() + builder.addStroke(stroke) + wheelActionsQueue.offer(builder.build()) + consumeWheelActions() + + } + + if (mask == WHEEL_UP) { + if (mouseY < WHEEL_STEP) { + return + } + val path = Path() + path.moveTo(mouseX.toFloat(), mouseY.toFloat()) + path.lineTo(mouseX.toFloat(), (mouseY + WHEEL_STEP).toFloat()) + val stroke = GestureDescription.StrokeDescription( + path, + 0, + WHEEL_DURATION + ) + val builder = GestureDescription.Builder() + builder.addStroke(stroke) + wheelActionsQueue.offer(builder.build()) + consumeWheelActions() + } + } + + @RequiresApi(Build.VERSION_CODES.N) + private fun consumeWheelActions() { + if (isWheelActionsPolling) { + return + } else { + isWheelActionsPolling = true + } + wheelActionsQueue.poll()?.let { + dispatchGesture(it, null, null) + timer.purge() + timer.schedule(object : TimerTask() { + override fun run() { + isWheelActionsPolling = false + consumeWheelActions() + } + }, WHEEL_DURATION + 10) + } ?: let { + isWheelActionsPolling = false + return } } private fun startGesture(x: Int, y: Int) { - mPath = Path() - mPath.moveTo(x.toFloat(), y.toFloat()) - mLastGestureStartTime = System.currentTimeMillis() + touchPath = Path() + touchPath.moveTo(x.toFloat(), y.toFloat()) + lastTouchGestureStartTime = System.currentTimeMillis() } private fun continueGesture(x: Int, y: Int) { - mPath.lineTo(x.toFloat(), y.toFloat()) + touchPath.lineTo(x.toFloat(), y.toFloat()) } @RequiresApi(Build.VERSION_CODES.N) private fun endGesture(x: Int, y: Int) { try { - mPath.lineTo(x.toFloat(), y.toFloat()) - var duration = System.currentTimeMillis() - mLastGestureStartTime + touchPath.lineTo(x.toFloat(), y.toFloat()) + var duration = System.currentTimeMillis() - lastTouchGestureStartTime if (duration <= 0) { duration = 1 } val stroke = GestureDescription.StrokeDescription( - mPath, + touchPath, 0, duration ) diff --git a/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt b/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt index fa79800b6..b08604ca3 100644 --- a/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt +++ b/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt @@ -9,7 +9,6 @@ import android.media.projection.MediaProjectionManager import android.os.Build import android.os.IBinder import android.provider.Settings -import android.util.DisplayMetrics import android.util.Log import androidx.annotation.RequiresApi import io.flutter.embedding.android.FlutterActivity @@ -31,7 +30,11 @@ class MainActivity : FlutterActivity() { @RequiresApi(Build.VERSION_CODES.M) override fun configureFlutterEngine(flutterEngine: FlutterEngine) { super.configureFlutterEngine(flutterEngine) - updateMachineInfo() + if (MainService.isReady) { + Intent(activity, MainService::class.java).also { + bindService(it, serviceConnection, Context.BIND_AUTO_CREATE) + } + } flutterMethodChannel = MethodChannel( flutterEngine.dartExecutor.binaryMessenger, channelTag @@ -43,7 +46,7 @@ class MainActivity : FlutterActivity() { Intent(activity, MainService::class.java).also { bindService(it, serviceConnection, Context.BIND_AUTO_CREATE) } - if (mainService?.isReady == true) { + if (MainService.isReady) { result.success(false) return@setMethodCallHandler } @@ -95,7 +98,7 @@ class MainActivity : FlutterActivity() { ) flutterMethodChannel.invokeMethod( "on_state_changed", - mapOf("name" to "media", "value" to mainService?.isReady.toString()) + mapOf("name" to "media", "value" to MainService.isReady.toString()) ) result.success(true) } @@ -190,37 +193,6 @@ class MainActivity : FlutterActivity() { } } - private fun updateMachineInfo() { - val dm = DisplayMetrics() - @Suppress("DEPRECATION") - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { - display?.getRealMetrics(dm) - } else { - windowManager.defaultDisplay.getRealMetrics(dm) - } - var w = dm.widthPixels - var h = dm.heightPixels - var scale = 1 - if (w != 0 && h != 0) { - if (w > MAX_SCREEN_SIZE || h > MAX_SCREEN_SIZE) { - scale = 2 - w /= scale - h /= scale - } - - INFO.screenWidth = w - INFO.screenHeight = h - INFO.scale = scale - INFO.username = "test" - INFO.hostname = "hostname" - // TODO username hostname - Log.d(logTag, "INIT INFO:$INFO") - - } else { - Log.e(logTag, "Got Screen Size Fail!") - } - } - override fun onDestroy() { Log.e(logTag, "onDestroy") mainService?.let { diff --git a/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index 8cbbb4eb4..391a3bccf 100644 --- a/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -12,6 +12,7 @@ import android.app.PendingIntent.FLAG_UPDATE_CURRENT import android.content.Context import android.content.Intent import android.content.pm.PackageManager +import android.content.res.Configuration import android.graphics.Color import android.graphics.PixelFormat import android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC @@ -20,9 +21,11 @@ import android.media.* import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.os.* +import android.util.DisplayMetrics import android.util.Log import android.view.Surface import android.view.Surface.FRAME_RATE_COMPATIBILITY_DEFAULT +import android.view.WindowManager import androidx.annotation.Keep import androidx.annotation.RequiresApi import androidx.core.app.ActivityCompat @@ -44,8 +47,6 @@ const val DEFAULT_NOTIFY_TEXT = "Service is running" const val DEFAULT_NOTIFY_ID = 1 const val NOTIFY_ID_OFFSET = 100 -const val NOTIFY_TYPE_START_CAPTURE = "NOTIFY_TYPE_START_CAPTURE" - const val MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_VP9 // video const @@ -68,7 +69,7 @@ class MainService : Service() { @Keep fun rustGetByName(name: String): String { return when (name) { - "screen_size" -> "${INFO.screenWidth}:${INFO.screenHeight}" + "screen_size" -> "${SCREEN_INFO.width}:${SCREEN_INFO.height}" else -> "" } } @@ -127,8 +128,10 @@ class MainService : Service() { private external fun init(ctx: Context) private external fun startServer() private external fun onVideoFrameUpdate(buf: ByteBuffer) + private external fun releaseVideoFrame() private external fun onAudioFrameUpdate(buf: ByteBuffer) private external fun translateLocale(localeName: String, input: String): String + private external fun refreshScreen() // private external fun sendVp9(data: ByteArray) private fun translate(input: String): String { @@ -136,15 +139,19 @@ class MainService : Service() { return translateLocale(LOCAL_NAME, input) } + companion object { + private var _isReady = false + private var _isStart = false + val isReady: Boolean + get() = _isReady + val isStart: Boolean + get() = _isStart + } + private val logTag = "LOG_SERVICE" private val useVP9 = false private val binder = LocalBinder() - private var _isReady = false - private var _isStart = false - val isReady: Boolean - get() = _isReady - val isStart: Boolean - get() = _isStart + // video private var mediaProjection: MediaProjection? = null @@ -167,10 +174,58 @@ class MainService : Service() { override fun onCreate() { super.onCreate() + updateScreenInfo() initNotification() startServer() } + override fun onDestroy() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + InputService.ctx?.disableSelf() + } + InputService.ctx = null + checkMediaPermission() + super.onDestroy() + } + + private fun updateScreenInfo() { + var w: Int + var h: Int + val windowManager = getSystemService(Context.WINDOW_SERVICE) as WindowManager + + @Suppress("DEPRECATION") + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + val m = windowManager.maximumWindowMetrics + w = m.bounds.width() + h = m.bounds.height() + } else { + val dm = DisplayMetrics() + windowManager.defaultDisplay.getRealMetrics(dm) + w = dm.widthPixels + h = dm.heightPixels + } + + var scale = 1 + if (w != 0 && h != 0) { + if (w > MAX_SCREEN_SIZE || h > MAX_SCREEN_SIZE) { + scale = 2 + w /= scale + h /= scale + } + if (SCREEN_INFO.width != w) { + SCREEN_INFO.width = w + SCREEN_INFO.height = h + SCREEN_INFO.scale = scale + if (isStart) { + stopCapture() + refreshScreen() + startCapture() + } + } + + } + } + override fun onBind(intent: Intent): IBinder { Log.d(logTag, "service onBind") return binder @@ -195,28 +250,29 @@ class MainService : Service() { mediaProjection = mMediaProjectionManager.getMediaProjection(Activity.RESULT_OK, it) checkMediaPermission() - surface = createSurface() init(this) _isReady = true - } ?: let { } -// } else if (intent?.action == ACTION_LOGIN_REQ_NOTIFY) { -// val notifyLoginRes = intent.getBooleanExtra(EXTRA_LOGIN_REQ_NOTIFY, false) } return super.onStartCommand(intent, flags, startId) } + override fun onConfigurationChanged(newConfig: Configuration) { + super.onConfigurationChanged(newConfig) + updateScreenInfo() + } + @SuppressLint("WrongConstant") private fun createSurface(): Surface? { return if (useVP9) { // TODO null } else { - Log.d(logTag, "ImageReader.newInstance:INFO:$INFO") + Log.d(logTag, "ImageReader.newInstance:INFO:$SCREEN_INFO") imageReader = ImageReader.newInstance( - INFO.screenWidth, - INFO.screenHeight, + SCREEN_INFO.width, + SCREEN_INFO.height, PixelFormat.RGBA_8888, 4 ).apply { @@ -247,6 +303,7 @@ class MainService : Service() { return false } Log.d(logTag, "Start Capture") + surface = createSurface() if (useVP9) { startVP9VideoRecorder(mediaProjection!!) @@ -266,6 +323,9 @@ class MainService : Service() { Log.d(logTag, "Stop Capture") _isStart = false // release video + imageReader?.close() + releaseVideoFrame() + surface?.release() virtualDisplay?.release() videoEncoder?.let { it.signalEndOfInputStream() @@ -292,7 +352,10 @@ class MainService : Service() { mediaProjection = null checkMediaPermission() - stopService(Intent(this, InputService::class.java)) // close input service maybe not work + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + InputService.ctx?.disableSelf() + } + InputService.ctx = null stopForeground(true) stopSelf() } @@ -304,19 +367,25 @@ class MainService : Service() { mapOf("name" to "media", "value" to isReady.toString()) ) } + Handler(Looper.getMainLooper()).post { + MainActivity.flutterMethodChannel.invokeMethod( + "on_state_changed", + mapOf("name" to "input", "value" to InputService.isOpen.toString()) + ) + } return isReady } @SuppressLint("WrongConstant") private fun startRawVideoRecorder(mp: MediaProjection) { - Log.d(logTag, "startRawVideoRecorder,screen info:$INFO") + Log.d(logTag, "startRawVideoRecorder,screen info:$SCREEN_INFO") if (surface == null) { Log.d(logTag, "startRawVideoRecorder failed,surface is null") return } virtualDisplay = mp.createVirtualDisplay( "RustDesk", - INFO.screenWidth, INFO.screenHeight, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC, + SCREEN_INFO.width, SCREEN_INFO.height, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC, surface, null, null ) } @@ -333,7 +402,7 @@ class MainService : Service() { it.start() virtualDisplay = mp.createVirtualDisplay( "RustDeskVD", - INFO.screenWidth, INFO.screenHeight, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC, + SCREEN_INFO.width, SCREEN_INFO.height, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC, surface, null, null ) } @@ -367,7 +436,8 @@ class MainService : Service() { private fun createMediaCodec() { Log.d(logTag, "MediaFormat.MIMETYPE_VIDEO_VP9 :$MIME_TYPE") videoEncoder = MediaCodec.createEncoderByType(MIME_TYPE) - val mFormat = MediaFormat.createVideoFormat(MIME_TYPE, INFO.screenWidth, INFO.screenHeight) + val mFormat = + MediaFormat.createVideoFormat(MIME_TYPE, SCREEN_INFO.width, SCREEN_INFO.height) mFormat.setInteger(MediaFormat.KEY_BIT_RATE, VIDEO_KEY_BIT_RATE) mFormat.setInteger(MediaFormat.KEY_FRAME_RATE, VIDEO_KEY_FRAME_RATE) mFormat.setInteger( diff --git a/android/app/src/main/kotlin/com/carriez/flutter_hbb/common.kt b/android/app/src/main/kotlin/com/carriez/flutter_hbb/common.kt index b18824964..895a7c658 100644 --- a/android/app/src/main/kotlin/com/carriez/flutter_hbb/common.kt +++ b/android/app/src/main/kotlin/com/carriez/flutter_hbb/common.kt @@ -18,12 +18,10 @@ import java.util.* @SuppressLint("ConstantLocale") val LOCAL_NAME = Locale.getDefault().toString() - -val INFO = Info("", "", 0, 0) +val SCREEN_INFO = Info(0, 0) data class Info( - var username: String, var hostname: String, var screenWidth: Int, var screenHeight: Int, - var scale: Int = 1 + var width: Int, var height: Int, var scale: Int = 1 ) @RequiresApi(Build.VERSION_CODES.LOLLIPOP) @@ -33,8 +31,8 @@ fun testVP9Support(): Boolean { .findEncoderForFormat( MediaFormat.createVideoFormat( MediaFormat.MIMETYPE_VIDEO_VP9, - INFO.screenWidth, - INFO.screenWidth + SCREEN_INFO.width, + SCREEN_INFO.width ) ) return res != null diff --git a/lib/common.dart b/lib/common.dart index ef0ced2b0..31d747d78 100644 --- a/lib/common.dart +++ b/lib/common.dart @@ -248,6 +248,8 @@ class AccessibilityListener extends StatelessWidget { pointer: evt.pointer + offset, size: 0.1, position: evt.position)); + GestureBinding.instance!.handlePointerEvent(PointerRemovedEvent( + pointer: evt.pointer + offset, position: evt.position)); } }, onPointerMove: (evt) { diff --git a/lib/models/server_model.dart b/lib/models/server_model.dart index 7a0ad2fa7..9561f43a0 100644 --- a/lib/models/server_model.dart +++ b/lib/models/server_model.dart @@ -182,6 +182,7 @@ class ServerModel with ChangeNotifier { await FFI.invokeMethod("init_service"); FFI.setByName("start_service"); getIDPasswd(); + updateClientState(); } Future stopService() async { @@ -281,12 +282,13 @@ class ServerModel with ChangeNotifier { try { final List clientsJson = jsonDecode(res); for (var clientJson in clientsJson) { - final client = Client.fromJson(jsonDecode(clientJson)); + final client = Client.fromJson(clientJson); _clients[client.id] = client; } - notifyListeners(); - } catch (e) {} + } catch (e) { + debugPrint("Failed to updateClientState:$e"); + } } loginRequest(Map evt) { diff --git a/pubspec.lock b/pubspec.lock index 056512dca..334ff42e9 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -7,7 +7,7 @@ packages: name: archive url: "https://pub.dartlang.org" source: hosted - version: "3.1.11" + version: "3.3.0" args: dependency: transitive description: @@ -161,7 +161,7 @@ packages: name: firebase_core url: "https://pub.dartlang.org" source: hosted - version: "1.12.0" + version: "1.14.1" firebase_core_platform_interface: dependency: transitive description: @@ -246,7 +246,7 @@ packages: name: image url: "https://pub.dartlang.org" source: hosted - version: "3.1.1" + version: "3.1.3" intl: dependency: transitive description: @@ -309,21 +309,21 @@ packages: name: path_provider url: "https://pub.dartlang.org" source: hosted - version: "2.0.8" + version: "2.0.9" path_provider_android: dependency: transitive description: name: path_provider_android url: "https://pub.dartlang.org" source: hosted - version: "2.0.11" + version: "2.0.12" path_provider_ios: dependency: transitive description: name: path_provider_ios url: "https://pub.dartlang.org" source: hosted - version: "2.0.7" + version: "2.0.8" path_provider_linux: dependency: transitive description: @@ -421,28 +421,28 @@ packages: name: shared_preferences_android url: "https://pub.dartlang.org" source: hosted - version: "2.0.10" + version: "2.0.11" shared_preferences_ios: dependency: transitive description: name: shared_preferences_ios url: "https://pub.dartlang.org" source: hosted - version: "2.0.9" + version: "2.1.0" shared_preferences_linux: dependency: transitive description: name: shared_preferences_linux url: "https://pub.dartlang.org" source: hosted - version: "2.0.4" + version: "2.1.0" shared_preferences_macos: dependency: transitive description: name: shared_preferences_macos url: "https://pub.dartlang.org" source: hosted - version: "2.0.2" + version: "2.0.3" shared_preferences_platform_interface: dependency: transitive description: @@ -463,7 +463,7 @@ packages: name: shared_preferences_windows url: "https://pub.dartlang.org" source: hosted - version: "2.0.4" + version: "2.1.0" sky_engine: dependency: transitive description: flutter @@ -545,7 +545,7 @@ packages: name: url_launcher url: "https://pub.dartlang.org" source: hosted - version: "6.0.18" + version: "6.0.20" url_launcher_android: dependency: transitive description: @@ -566,14 +566,14 @@ packages: name: url_launcher_linux url: "https://pub.dartlang.org" source: hosted - version: "2.0.3" + version: "3.0.0" url_launcher_macos: dependency: transitive description: name: url_launcher_macos url: "https://pub.dartlang.org" source: hosted - version: "2.0.3" + version: "3.0.0" url_launcher_platform_interface: dependency: transitive description: @@ -587,14 +587,14 @@ packages: name: url_launcher_web url: "https://pub.dartlang.org" source: hosted - version: "2.0.8" + version: "2.0.9" url_launcher_windows: dependency: transitive description: name: url_launcher_windows url: "https://pub.dartlang.org" source: hosted - version: "2.0.2" + version: "3.0.0" uuid: dependency: transitive description: @@ -650,7 +650,7 @@ packages: name: win32 url: "https://pub.dartlang.org" source: hosted - version: "2.3.11" + version: "2.5.1" xdg_directories: dependency: transitive description: diff --git a/web/index.html b/web/index.html index 906f95158..83a24c4e7 100644 --- a/web/index.html +++ b/web/index.html @@ -154,8 +154,8 @@ loadMainDartJs(); } - - + +