android update,open and close service

android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -5,6 +5,8 @@ import android.accessibilityservice.GestureDescription
 import android.content.Context
 import android.graphics.Path
 import android.os.Build
+import android.os.Handler
+import android.os.Looper
 import android.util.Log
 import android.view.accessibility.AccessibilityEvent
 import androidx.annotation.RequiresApi
@@ -12,6 +14,12 @@ import kotlin.concurrent.thread
 
 class InputService : AccessibilityService() {
 
+    companion object{
+        var ctx:InputService? = null
+        fun isOpen():Boolean{
+            return ctx!=null
+        }
+    }
     private val logTag = "input service"
     private var leftIsDown = false
     private var mPath = Path()
@@ -21,28 +29,26 @@ class InputService : AccessibilityService() {
 
     @RequiresApi(Build.VERSION_CODES.N)
     fun rustMouseInput(mask: Int, _x: Int, _y: Int) {
-        Log.w(logTag, "got mouse input:x:$_x ,y:$_y ,mask:$mask ")
 
-        // TODO 临时倍数
         // TODO 按键抬起按下时候 x y 都是0
-        if ( !(mask == 9 || mask == 10) ) {
+        if (!(mask == 9 || mask == 10)) {
-            mouseX = _x * SCALE
+            mouseX = _x * INFO.scale
-            mouseY = _y * SCALE
+            mouseY = _y * INFO.scale
         }
 
         // left button down ,was up
-        if (mask == 9){
+        if (mask == 9) {
             leftIsDown = true
-            startGesture(mouseX,mouseY)
+            startGesture(mouseX, mouseY)
         }
 
         // left down ,was down
-        if (mask == 9){
+        if (mask == 9) {
-            continueGesture(mouseX,mouseY)
+            continueGesture(mouseX, mouseY)
         }
 
         // left up ,was down
-        if (mask == 10){
+        if (mask == 10) {
             leftIsDown = false
             endGesture(mouseX, mouseY)
         }
@@ -57,6 +63,7 @@ class InputService : AccessibilityService() {
     private fun continueGesture(x: Int, y: Int) {
         mPath.lineTo(x.toFloat(), y.toFloat())
     }
+
     @RequiresApi(Build.VERSION_CODES.N)
     private fun endGesture(x: Int, y: Int) {
         mPath.lineTo(x.toFloat(), y.toFloat())
@@ -81,19 +88,20 @@ class InputService : AccessibilityService() {
         }, null)
     }
 
-    external fun init(ctx: Context)
+    private external fun init(ctx: Context)
 
     init {
         System.loadLibrary("rustdesk")
     }
 
-    private val LOG_TAG = "INPUT_LOG"
     @RequiresApi(Build.VERSION_CODES.O)
     override fun onServiceConnected() {
         super.onServiceConnected()
-        Log.d(LOG_TAG,"onServiceConnected!")
+        ctx = this
+        Log.d(logTag, "onServiceConnected!")
         init(this)
     }
+
     override fun onAccessibilityEvent(event: AccessibilityEvent?) {
 //        TODO("Not yet implemented")
     }

android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt

@@ -1,7 +1,7 @@
 package com.carriez.flutter_hbb
 
 import android.app.Activity
-import android.app.AlertDialog
+import android.content.Context
 import android.content.Intent
 import android.media.projection.MediaProjectionManager
 import android.os.Build
@@ -9,126 +9,220 @@ import android.provider.Settings
 import android.util.DisplayMetrics
 import android.util.Log
 import androidx.annotation.RequiresApi
+import androidx.core.app.NotificationManagerCompat
 import io.flutter.embedding.android.FlutterActivity
 import io.flutter.embedding.engine.FlutterEngine
 import io.flutter.plugin.common.MethodChannel
 
-const val MAX_SIZE = 1400
+const val NOTIFY_TYPE_LOGIN_REQ = "NOTIFY_TYPE_LOGIN_REQ"
+const val MEDIA_REQUEST_CODE = 42
+const val INPUT_REQUEST_CODE = 43
 
 class MainActivity : FlutterActivity() {
+    companion object {
+        lateinit var flutterMethodChannel: MethodChannel
+    }
+
     private val channelTag = "mChannel"
     private val logTag = "mMainActivity"
     private var mediaProjectionResultIntent: Intent? = null
-    private val requestCode = 1
 
     init {
         System.loadLibrary("rustdesk")
     }
 
-    external fun rustSetInfo(username: String, hostname: String, width: Int, height: Int)
+    private external fun init(context: Context)
+    private external fun close()
 
-    override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
+    fun rustSetByName(name: String, arg1: String, arg2: String) {
-        super.configureFlutterEngine(flutterEngine) // 必要 否则无法正确初始化flutter
+        when (name) {
-        updateMachineInfo()
+            "try_start_without_auth" -> {
-        MethodChannel(
+                // to UI
-            flutterEngine.dartExecutor.binaryMessenger,
+                Log.d(logTag, "from rust:got try_start_without_auth")
-            channelTag
+                activity.runOnUiThread {
-        ).setMethodCallHandler { call, result ->
+                    flutterMethodChannel.invokeMethod(name, mapOf("peerID" to arg1, "name" to arg2))
-            when (call.method) {
+                    Log.d(logTag, "activity.runOnUiThread invokeMethod try_start_without_auth,done")
-                "getPer" -> {
-                    Log.d(channelTag, "event from flutter,getPer")
-                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
-                        getMediaProjection()
-                    }
-                    result.success(true)
                 }
-                "startSer" -> {
+                val notification = createNormalNotification(
-                    mStarService()
+                    this,
-                    result.success(true)
+                    "请求控制",
+                    "来自$arg1:$arg2 请求连接",
+                    NOTIFY_TYPE_LOGIN_REQ
+                )
+                with(NotificationManagerCompat.from(this)) {
+                    notify(12, notification)
                 }
-                "stopSer" -> {
+                Log.d(logTag, "kotlin invokeMethod try_start_without_auth,done")
-                    mStopService()
-                    result.success(true)
-                }
-                "checkInput" -> {
-                    checkInput()
-                    result.success(true)
-                }
-                else -> {}
             }
+            "start_capture" -> {
+                Log.d(logTag, "from rust:start_capture")
+                activity.runOnUiThread {
+                    flutterMethodChannel.invokeMethod(name, mapOf("peerID" to arg1, "name" to arg2))
+                    Log.d(logTag, "activity.runOnUiThread invokeMethod try_start_without_auth,done")
+                }
+                // 1.开始捕捉音视频 2.通知栏
+                startCapture()
+                val notification = createNormalNotification(
+                    this,
+                    "开始共享屏幕",
+                    "From:$arg2:$arg1",
+                    NOTIFY_TYPE_START_CAPTURE
+                )
+                with(NotificationManagerCompat.from(this)) {
+                    notify(13, notification)
+                }
+            }
+            "stop_capture" -> {
+                Log.d(logTag, "from rust:stop_capture")
+                stopCapture()
+                activity.runOnUiThread {
+                    flutterMethodChannel.invokeMethod(name, null)
+                    Log.d(logTag, "activity.runOnUiThread invokeMethod try_start_without_auth,done")
+                }
+            }
+            else -> {}
         }
     }
 
+    override fun onDestroy() {
+        Log.e(logTag, "onDestroy")
+        close()
+        stopCapture()
+        stopMainService()
+        stopService(Intent(this, MainService::class.java))
+        stopService(Intent(this, InputService::class.java))
+        super.onDestroy()
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
+        super.configureFlutterEngine(flutterEngine) // 必要 否则无法正确初始化flutter
+        checkPermissions(this)
+        updateMachineInfo()
+        flutterMethodChannel = MethodChannel(
+            flutterEngine.dartExecutor.binaryMessenger,
+            channelTag
+        ).apply {
+            setMethodCallHandler { call, result ->
+                when (call.method) {
+                    "init_service" -> {
+                        Log.d(logTag, "event from flutter,getPer")
+                        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+                            getMediaProjection()
+                        }
+                        result.success(true)
+                    }
+                    "start_capture" -> {
+                        startCapture()
+                        result.success(true)
+                    }
+                    "stop_service" -> {
+                        stopMainService()
+                        result.success(true)
+                    }
+                    "check_input" -> {
+                        checkInput()
+                        result.success(true)
+                    }
+                    "check_video_permission" -> {
+                        val res = MainService.checkMediaPermission()
+                        result.success(res)
+                    }
+                    else -> {}
+                }
+            }
+        }
+    }
 
     @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
     private fun getMediaProjection() {
         val mMediaProjectionManager =
             getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
         val mIntent = mMediaProjectionManager.createScreenCaptureIntent()
-        startActivityForResult(mIntent, requestCode)
+        startActivityForResult(mIntent, MEDIA_REQUEST_CODE)
     }
 
-    private fun mStarService() {
+    // 实际逻辑是开始监听服务 在成功获取到mediaProjection就开始
+    private fun initService() {
         if (mediaProjectionResultIntent == null) {
-            Log.w(channelTag, "mediaProjectionResultIntent is null")
+            Log.w(logTag, "initService fail,mediaProjectionResultIntent is null")
             return
         }
-        Log.d(channelTag, "Start a service")
+        Log.d(logTag, "Init service")
+        // call init service to rust
+        init(this)
         val serviceIntent = Intent(this, MainService::class.java)
-        serviceIntent.action = START_SERVICE
+        serviceIntent.action = INIT_SERVICE
         serviceIntent.putExtra(EXTRA_MP_DATA, mediaProjectionResultIntent)
 
-        // TEST api < O
+        launchMainService(serviceIntent)
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
-            startForegroundService(serviceIntent)
-        } else {
-            startService(serviceIntent)
-        }
     }
 
-    private fun mStopService() {
+    private fun startCapture() {
-        Log.d(channelTag, "Stop service")
+        if (mediaProjectionResultIntent == null) {
+            Log.w(logTag, "startCapture fail,mediaProjectionResultIntent is null")
+            return
+        }
+        Log.d(logTag, "Start Capture")
         val serviceIntent = Intent(this, MainService::class.java)
+        serviceIntent.action = START_CAPTURE
+        serviceIntent.putExtra(EXTRA_MP_DATA, mediaProjectionResultIntent)
 
+        launchMainService(serviceIntent)
+    }
+
+    private fun stopCapture() {
+        Log.d(logTag, "Stop Capture")
+        val serviceIntent = Intent(this, MainService::class.java)
+        serviceIntent.action = STOP_CAPTURE
+
+        launchMainService(serviceIntent)
+    }
+
+    // TODO 关闭逻辑
+    private fun stopMainService() {
+        Log.d(logTag, "Stop service")
+        val serviceIntent = Intent(this, MainService::class.java)
         serviceIntent.action = STOP_SERVICE
+        launchMainService(serviceIntent)
+    }
 
+    private fun launchMainService(intent: Intent) {
         // TEST api < O
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
-            startForegroundService(serviceIntent)
+            startForegroundService(intent)
         } else {
-            startService(serviceIntent)
+            startService(intent)
         }
     }
 
     private fun checkInput() {
-        AlertDialog.Builder(this)
+        val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)
-            .setCancelable(false)
+        if (intent.resolveActivity(packageManager) != null) {
-            .setTitle("检查Input服务")
+            startActivity(intent)
-            .setMessage("请开启相关服务")
+        }
-            .setPositiveButton("Yes") { dialog, which ->
+    }
-                val intent = Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)
+
-                if (intent.resolveActivity(packageManager) != null) startActivityForResult(
+    override fun onResume() {
-                    intent,
+        super.onResume()
-                    11
+        val inputPer = InputService.isOpen()
-                ) else AlertDialog.Builder(this)
+        Log.d(logTag,"onResume inputPer:$inputPer")
-                    .setTitle("错误")
+        activity.runOnUiThread {
-                    .setMessage("无法启动服务")
+            flutterMethodChannel.invokeMethod("on_permission_changed",mapOf("name" to "input", "value" to inputPer.toString()))
-                    .show()
+        }
-            }
-            .setNegativeButton("No") { dialog, which -> }
-            .show()
     }
 
     override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
         super.onActivityResult(requestCode, resultCode, data)
-        if (resultCode == Activity.RESULT_OK && data != null) {
+        if (requestCode == MEDIA_REQUEST_CODE && resultCode == Activity.RESULT_OK && data != null) {
-            Log.d(channelTag, "got mediaProjectionResultIntent ok")
+            Log.d(logTag, "got mediaProjectionResultIntent ok")
             mediaProjectionResultIntent = data
+            initService()
         }
     }
 
     private fun updateMachineInfo() {
-        // 屏幕尺寸 控制最长边不超过1400 超过则减半直到1400 并储存缩放比例 实际发送给手机端的尺寸为缩小后的尺寸
+        // 屏幕尺寸 控制最长边不超过1400 超过则减半并储存缩放比例 实际发送给手机端的尺寸为缩小后的尺寸
         // input控制时再通过缩放比例恢复原始尺寸进行path入参
         val dm = DisplayMetrics()
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
@@ -146,18 +240,20 @@ class MainActivity : FlutterActivity() {
         var h = dm.heightPixels
         var scale = 1
         if (w != 0 && h != 0) {
-            if (w > MAX_SIZE || h > MAX_SIZE) {
+            if (w > MAX_SCREEN_SIZE || h > MAX_SCREEN_SIZE) {
                 scale = 2
                 w /= scale
                 h /= scale
             }
             Log.d(logTag, "Real size - width:$w,height:$h")
 
-            FIXED_WIDTH = 540
+            INFO.screenWidth = w
-            FIXED_HEIGHT = 1140
+            INFO.screenHeight = h
-            SCALE = scale
+            INFO.scale = scale
+            INFO.username = "test"
+            INFO.hostname = "hostname"
             // TODO  username hostname
-            rustSetInfo("csf", "Android", FIXED_WIDTH, FIXED_HEIGHT)
+
         } else {
             Log.e(logTag, "Got Screen Size Fail!")
         }

android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -1,156 +1,292 @@
+/**
+ * video_service and audio_service
+ */
 package com.carriez.flutter_hbb
 
 import android.annotation.SuppressLint
 import android.app.*
 import android.content.Context
 import android.content.Intent
-import android.graphics.Color
 import android.graphics.PixelFormat
 import android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC
+import android.hardware.display.VirtualDisplay
 import android.media.*
+import android.media.AudioRecord.READ_BLOCKING
 import android.media.projection.MediaProjection
 import android.media.projection.MediaProjectionManager
 import android.os.Build
+import android.os.Handler
 import android.os.IBinder
-import android.util.DisplayMetrics
+import android.os.Looper
 import android.util.Log
 import android.view.Surface
-import android.view.WindowManager
+import android.view.Surface.FRAME_RATE_COMPATIBILITY_DEFAULT
 import android.widget.Toast
 import androidx.annotation.RequiresApi
-import androidx.core.app.NotificationCompat
-import androidx.core.app.NotificationCompat.PRIORITY_MIN
-import java.nio.ByteBuffer
 import java.util.concurrent.Executors
+import kotlin.concurrent.thread
 
 const val EXTRA_MP_DATA = "mp_intent"
-const val START_SERVICE = "start_service"
+const val INIT_SERVICE = "init_service"
+const val START_CAPTURE = "start_capture"
+const val STOP_CAPTURE = "stop_capture"
 const val STOP_SERVICE = "stop_service"
+
+const val NOTIFY_TYPE_START_CAPTURE = "NOTIFY_TYPE_START_CAPTURE"
+
+@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
 const val MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_VP9
 
-var FIXED_WIDTH = 0 // 编码器有上限
+// video const
-var FIXED_HEIGHT = 0
+const val MAX_SCREEN_SIZE = 1200 // 内置编码器有上限 且实际使用中不需要过高的分辨率
-var SCALE = 1  // RealScreenWidth = screenWidth * scale
+
-const val M_KEY_BIT_RATE = 1024_000
+const val VIDEO_KEY_BIT_RATE = 1024_000
-const val M_KEY_FRAME_RATE = 30
+const val VIDEO_KEY_FRAME_RATE = 30
+
+// audio const
+@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+const val AUDIO_ENCODING = AudioFormat.ENCODING_PCM_FLOAT //  ENCODING_OPUS need API 30
+const val AUDIO_SAMPLE_RATE = 48000
+const val AUDIO_CHANNEL_MASK = AudioFormat.CHANNEL_IN_STEREO
 
 class MainService : Service() {
 
-    fun rustGetRaw(): ByteArray {
+    companion object {
-        return rawByteArray!!
+        private var mediaProjection: MediaProjection? = null
+        fun checkMediaPermission(): Boolean {
+            val value = mediaProjection != null
+            Handler(Looper.getMainLooper()).post {
+                MainActivity.flutterMethodChannel.invokeMethod(
+                    "on_permission_changed",
+                    mapOf("name" to "media", "value" to value.toString())
+                )
+            }
+            return value
+        }
     }
 
-    external fun init(ctx: Context)
-
     init {
         System.loadLibrary("rustdesk")
     }
 
-    private val logTag = "LOG_SERVICE"
+    // rust call jvm
-    private var mMediaProjection: MediaProjection? = null
+    fun rustGetVideoRaw(): ByteArray {
-    private var surface: Surface? = null
+        return if (videoData != null) {
-    private val singleThread = Executors.newSingleThreadExecutor()
+            videoData!!
-    private var mEncoder: MediaCodec? = null
+        } else {
-    private var rawByteArray: ByteArray? = null
+            videoZeroData
-
+        }
-    override fun onBind(intent: Intent): IBinder? {
-        return null
     }
 
+    fun rustGetAudioRaw(): FloatArray {
+        return if (isNewData && audioData != null) {
+            isNewData = false
+            audioData!!
+        } else {
+            audioZeroData
+        }
+    }
+
+    fun rustGetAudioRawLen(): Int {
+        return if (isNewData && audioData != null && audioData!!.isNotEmpty()) {
+            audioData!!.size
+        } else 0
+    }
+
+    fun rustGetByName(name: String): String {
+        return when (name) {
+            "screen_size" -> "${INFO.screenWidth}:${INFO.screenHeight}"
+            else -> ""
+        }
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    fun rustSetByName(name: String, arg1: String, arg2: String) {
+        when (name) {
+            else -> {}
+        }
+    }
+
+    // jvm call rust
+    private external fun init(ctx: Context)
+    private external fun sendVp9(data: ByteArray)
+
+    private val logTag = "LOG_SERVICE"
+    private val useVP9 = false
+
+    // video
+    private var surface: Surface? = null
+    private val sendVP9Thread = Executors.newSingleThreadExecutor()
+    private var videoEncoder: MediaCodec? = null
+    private var videoData: ByteArray? = null
+    private var imageReader: ImageReader? =
+        null // * 注意 这里要成为成员变量，防止被回收 https://www.cnblogs.com/yongdaimi/p/11004560.html
+    private val videoZeroData = ByteArray(32)
+    private var virtualDisplay: VirtualDisplay? = null
+
+    // audio
+    private var audioRecorder: AudioRecord? = null
+    private var audioData: FloatArray? = null
+    private var minBufferSize = 0
+    private var isNewData = false
+    private val audioZeroData: FloatArray = FloatArray(32)  // 必须是32位 如果只有8位进行ffi传输时会出错
+    private var audioRecordStat = false
+
     @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
     override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
         Log.d("whichService", "this service:${Thread.currentThread()}")
-        init(this) // 注册到rust
+        when (intent?.action) {
-        if (intent?.action == START_SERVICE) {
+            INIT_SERVICE -> {
-            Log.d(logTag, "service starting:${startId}:${Thread.currentThread()}")
+                Log.d(logTag, "service starting:${startId}:${Thread.currentThread()}")
-            createNotification()
+                createForegroundNotification(this)
-            val mMediaProjectionManager =
+                val mMediaProjectionManager =
-                getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
+                    getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
-            mMediaProjection = intent.getParcelableExtra<Intent>(EXTRA_MP_DATA)?.let {
+                intent.getParcelableExtra<Intent>(EXTRA_MP_DATA)?.let {
-                mMediaProjectionManager.getMediaProjection(Activity.RESULT_OK, it)
+                    mediaProjection =
-            }
+                        mMediaProjectionManager.getMediaProjection(Activity.RESULT_OK, it)
-            Log.d(logTag, "获取mMediaProjection成功$mMediaProjection")
+                    Log.d(logTag, "获取mMediaProjection成功$mediaProjection")
-            if (testSupport()) {
+                    checkMediaPermission()
-                startRecorder()
+                    init(this)
-            } else {
+                } ?: let {
-                Toast.makeText(this, "此设备不支持:$MIME_TYPE", Toast.LENGTH_SHORT).show()
+                    Log.d(logTag, "获取mMediaProjection失败！")
-                stopSelf(startId)
-            }
-        } else if (intent?.action == STOP_SERVICE) {
-            mEncoder?.let {
-                try {
-                    Log.d(logTag, "正在释放encoder")
-                    it.signalEndOfInputStream()
-                    it.stop()
-                    it.release()
-                } catch (e: Exception) {
-                    null
                 }
             }
-            stopSelf()
+            START_CAPTURE -> {
+                startCapture()
+            }
+            STOP_CAPTURE -> {
+                stopCapture()
+            }
+            STOP_SERVICE -> {
+                stopCapture()
+                mediaProjection = null
+                checkMediaPermission()
+                stopSelf()
+            }
         }
         return super.onStartCommand(intent, flags, startId)
     }
 
-    lateinit var mImageReader: ImageReader // * 注意 这里要成为成员变量，防止被回收 https://www.cnblogs.com/yongdaimi/p/11004560.html
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    private fun startCapture(): Boolean {
+        if (testVP9Support()) {  // testVP9Support一直返回true 暂时只使用原始数据
+            startVideoRecorder()
+        } else {
+            Toast.makeText(this, "此设备不支持:$MIME_TYPE", Toast.LENGTH_SHORT).show()
+            return false
+        }
+        // 音频只支持安卓10以及以上
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+            startAudioRecorder()
+        }
+        checkMediaPermission()
+        return true
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    private fun stopCapture() {
+        virtualDisplay?.release()
+        imageReader?.close()
+        videoEncoder?.let {
+            it.signalEndOfInputStream()
+            it.stop()
+            it.release()
+        }
+        audioRecorder?.startRecording()
+        audioRecordStat = false
+
+        // audioRecorder 如果无法重新创建 保留服务的情况不要释放
+//        audioRecorder?.stop()
+//        mediaProjection?.stop()
+
+        virtualDisplay = null
+        imageReader = null
+        videoEncoder = null
+        videoData = null
+//        audioRecorder = null
+//        audioData = null
+    }
+
 
     @SuppressLint("WrongConstant")
     @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
-    private fun startRecorder() {
+    private fun startVideoRecorder() {
-        Log.d(logTag, "startRecorder")
+        Log.d(logTag, "startVideoRecorder")
-        mMediaProjection?.let { mp ->
+        mediaProjection?.let { mp ->
-            // 使用原始数据
+            if (useVP9) {
-            mImageReader =
+                startVP9VideoRecorder(mp)
-                ImageReader.newInstance(FIXED_WIDTH, FIXED_HEIGHT, PixelFormat.RGBA_8888, 2) // 至少是2
+            } else {
-            mImageReader.setOnImageAvailableListener({ imageReader: ImageReader ->
+                startRawVideoRecorder(mp)
-//                Log.d(logTag, "on image")
+            }
-                try {
-                    imageReader.acquireLatestImage().use { image ->
-                        if (image == null) return@setOnImageAvailableListener
-                        val planes = image.planes
-                        val buffer = planes[0].buffer
-                        buffer.rewind()
-                        // 这里注意 处理不当会引发OOM
-                        if (rawByteArray == null) {
-                            rawByteArray = ByteArray(buffer.capacity())
-                            buffer.get(rawByteArray!!)
-                        } else {
-                            buffer.get(rawByteArray!!)
-                        }
-                    }
-                } catch (ignored: java.lang.Exception) {
-                }
-                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
-                    imageReader.discardFreeBuffers()
-                }
-            }, null)
-            mp.createVirtualDisplay(
-                "rustdesk test",
-                FIXED_WIDTH, FIXED_HEIGHT, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
-                mImageReader.surface, null, null
-            )
-
-
-            // 使用内置编码器
-//            createMediaCodec()
-//            mEncoder?.let {
-//                surface = it.createInputSurface()
-//                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
-//                    surface!!.setFrameRate(1F, FRAME_RATE_COMPATIBILITY_DEFAULT)
-//                }
-//                it.setCallback(cb)
-//                it.start()
-//                mp.createVirtualDisplay(
-//                    "rustdesk test",
-//                    FIXED_WIDTH, FIXED_HEIGHT, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
-//                    surface, null, null
-//                )
-//            }
         } ?: let {
             Log.d(logTag, "startRecorder fail,mMediaProjection is null")
         }
     }
 
+    @SuppressLint("WrongConstant")
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    private fun startRawVideoRecorder(mp: MediaProjection) {
+        Log.d(logTag, "startRawVideoRecorder,screen info:$INFO")
+        // 使用原始数据
+        imageReader =
+            ImageReader.newInstance(
+                INFO.screenWidth,
+                INFO.screenHeight,
+                PixelFormat.RGBA_8888,
+                2 // maxImages 至少是2
+            ).apply {
+                // 奇怪的现象，必须从MainActivity调用 无法从MainService中调用 会阻塞在这个函数
+                setOnImageAvailableListener({ imageReader: ImageReader ->
+                    try {
+                        imageReader.acquireLatestImage().use { image ->
+                            if (image == null) return@setOnImageAvailableListener
+                            val planes = image.planes
+                            val buffer = planes[0].buffer
+                            buffer.rewind()
+                            // Be careful about OOM!
+                            if (videoData == null) {
+                                videoData = ByteArray(buffer.capacity())
+                                buffer.get(videoData!!)
+                                Log.d(logTag, "init video ${videoData!!.size}")
+                            } else {
+                                buffer.get(videoData!!)
+                            }
+                        }
+                    } catch (ignored: java.lang.Exception) {
+                    }
+                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+                        imageReader.discardFreeBuffers()
+                    }
+                }, null)
+            }
+        Log.d(logTag, "ImageReader.setOnImageAvailableListener done")
+        virtualDisplay = mp.createVirtualDisplay(
+            "RustDesk",
+            INFO.screenWidth, INFO.screenHeight, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
+            imageReader?.surface, null, null
+        )
+        Log.d(logTag, "startRawVideoRecorder done")
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    private fun startVP9VideoRecorder(mp: MediaProjection) {
+        //使用内置编码器
+        createMediaCodec()
+        videoEncoder?.let {
+            surface = it.createInputSurface()
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+                surface!!.setFrameRate(1F, FRAME_RATE_COMPATIBILITY_DEFAULT)
+            }
+            it.setCallback(cb)
+            it.start()
+            virtualDisplay = mp.createVirtualDisplay(
+                "rustdesk test",
+                INFO.screenWidth, INFO.screenHeight, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
+                surface, null, null
+            )
+        }
+    }
+
     private val cb: MediaCodec.Callback = @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
     object : MediaCodec.Callback() {
         override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {}
@@ -162,7 +298,7 @@ class MainService : Service() {
             info: MediaCodec.BufferInfo
         ) {
             codec.getOutputBuffer(index)?.let { buf ->
-                singleThread.execute {
+                sendVP9Thread.execute {
                     // TODO 优化内存使用方式
                     val byteArray = ByteArray(buf.limit())
                     buf.get(byteArray)
@@ -177,75 +313,92 @@ class MainService : Service() {
         }
     }
 
-    external fun sendRaw(buf: ByteBuffer)
-    external fun sendVp9(data: ByteArray)
 
     @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
-    private fun testSupport(): Boolean {
-        val res = MediaCodecList(MediaCodecList.ALL_CODECS)
-            .findEncoderForFormat(
-                MediaFormat.createVideoFormat(
-                    MediaFormat.MIMETYPE_VIDEO_VP9,
-                    FIXED_WIDTH,
-                    FIXED_HEIGHT
-                )
-            )
-        return res?.let {
-            true
-        } ?: let {
-            false
-        }
-    }
-
     private fun createMediaCodec() {
         Log.d(logTag, "MediaFormat.MIMETYPE_VIDEO_VP9 :$MIME_TYPE")
-        mEncoder = MediaCodec.createEncoderByType(MIME_TYPE)
+        videoEncoder = MediaCodec.createEncoderByType(MIME_TYPE)
-        val mFormat = MediaFormat.createVideoFormat(MIME_TYPE, FIXED_WIDTH, FIXED_HEIGHT)
+        val mFormat = MediaFormat.createVideoFormat(MIME_TYPE, INFO.screenWidth, INFO.screenHeight)
-        mFormat.setInteger(MediaFormat.KEY_BIT_RATE, M_KEY_BIT_RATE)
+        mFormat.setInteger(MediaFormat.KEY_BIT_RATE, VIDEO_KEY_BIT_RATE)
-        mFormat.setInteger(MediaFormat.KEY_FRAME_RATE, M_KEY_FRAME_RATE) // codec的帧率设置无效
+        mFormat.setInteger(MediaFormat.KEY_FRAME_RATE, VIDEO_KEY_FRAME_RATE) // codec的帧率设置无效
         mFormat.setInteger(
             MediaFormat.KEY_COLOR_FORMAT,
             MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
         )
         mFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5)
         try {
-            mEncoder!!.configure(mFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
+            videoEncoder!!.configure(mFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
         } catch (e: Exception) {
             Log.e(logTag, "mEncoder.configure fail!")
         }
     }
 
-    private fun createNotification() {
+    @RequiresApi(Build.VERSION_CODES.M)
-        val channelId =
+    private fun startAudioRecorder() {
-            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+        checkAudioRecorder()
-                createNotificationChannel("my_service", "My Background Service")
+        if (audioData != null && audioRecorder != null && minBufferSize != 0) {
-            } else {
+            audioRecorder!!.startRecording()
-                ""
+            audioRecordStat = true
+            thread {
+                while (audioRecordStat) {
+                    val res = audioRecorder!!.read(audioData!!, 0, minBufferSize, READ_BLOCKING)
+                    // 录制float 需要使用对应的read float[] 函数
+                    if (res != AudioRecord.ERROR_INVALID_OPERATION) {
+                        isNewData = true
+                    }
+                }
+                Log.d(logTag, "Exit audio thread")
             }
-        val notification: Notification = NotificationCompat.Builder(this, channelId)
+        } else {
-            .setOngoing(true)
+            Log.d(logTag, "startAudioRecorder fail")
-            .setContentTitle("Hello")
+        }
-            .setPriority(PRIORITY_MIN)
-            .setContentText("TEST TEST")
-            .build()
-        startForeground(11, notification)
     }
 
-    @RequiresApi(Build.VERSION_CODES.O)
+    @RequiresApi(Build.VERSION_CODES.M)
-    private fun createNotificationChannel(channelId: String, channelName: String): String {
+    private fun checkAudioRecorder() {
-        val chan = NotificationChannel(
+        if (audioData != null && audioRecorder != null && minBufferSize != 0) {
-            channelId,
+            return
-            channelName, NotificationManager.IMPORTANCE_NONE
+        }
+        minBufferSize = 2 * AudioRecord.getMinBufferSize(
+            AUDIO_SAMPLE_RATE,
+            AUDIO_CHANNEL_MASK,
+            AUDIO_ENCODING
         )
-        chan.lightColor = Color.BLUE
+        if (minBufferSize == 0) {
-        chan.lockscreenVisibility = Notification.VISIBILITY_PRIVATE
+            Log.d(logTag, "get min buffer size fail!")
-        val service = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+            return
-        service.createNotificationChannel(chan)
+        }
-        return channelId
+        audioData = FloatArray(minBufferSize)
+        Log.d(logTag, "init audioData len:${audioData!!.size}")
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+            mediaProjection?.let {
+                val apcc = AudioPlaybackCaptureConfiguration.Builder(it)
+                    .addMatchingUsage(AudioAttributes.USAGE_MEDIA)
+                    .addMatchingUsage(AudioAttributes.USAGE_ALARM)
+                    .addMatchingUsage(AudioAttributes.USAGE_GAME)
+                    .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN).build()
+                audioRecorder = AudioRecord.Builder()
+                    .setAudioFormat(
+                        AudioFormat.Builder()
+                            .setEncoding(AUDIO_ENCODING)
+                            .setSampleRate(AUDIO_SAMPLE_RATE)
+                            .setChannelMask(AUDIO_CHANNEL_MASK).build()
+                    )
+                    .setAudioPlaybackCaptureConfig(apcc)
+                    .setBufferSizeInBytes(minBufferSize).build()
+                Log.d(logTag, "createAudioRecorder done,minBufferSize:$minBufferSize")
+                return
+            }
+        }
+        Log.d(logTag, "createAudioRecorder fail")
     }
 
     override fun onDestroy() {
         Log.d(logTag, "service stop:${Thread.currentThread()}")
         Toast.makeText(this, "service done", Toast.LENGTH_SHORT).show()
     }
+
+    override fun onBind(intent: Intent): IBinder? {
+        return null
+    }
 }

android/app/src/main/kotlin/com/carriez/flutter_hbb/common.kt

@@ -0,0 +1,129 @@
+package com.carriez.flutter_hbb
+
+import android.Manifest
+import android.app.*
+import android.app.PendingIntent.FLAG_UPDATE_CURRENT
+import android.content.Context
+import android.content.Intent
+import android.content.pm.PackageManager
+import android.graphics.Color
+import android.graphics.drawable.Icon
+import android.media.MediaCodecList
+import android.media.MediaFormat
+import android.os.Build
+import androidx.annotation.RequiresApi
+import androidx.core.app.ActivityCompat
+import androidx.core.app.NotificationCompat
+import androidx.core.content.ContextCompat
+import java.util.*
+
+val INFO = Info("","",0,0)
+
+data class Info(var username:String, var hostname:String, var screenWidth:Int, var screenHeight:Int,
+                var scale:Int = 1)
+
+
+@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+fun testVP9Support(): Boolean {
+    return true  // 函数内部永远返回true 暂时只使用原始数据
+    val res = MediaCodecList(MediaCodecList.ALL_CODECS)
+        .findEncoderForFormat(
+            MediaFormat.createVideoFormat(
+                MediaFormat.MIMETYPE_VIDEO_VP9,
+                INFO.screenWidth,
+                INFO.screenWidth
+            )
+        )
+    return res!=null
+}
+
+fun createForegroundNotification(ctx:Service) {
+    // 设置通知渠道 android8开始引入 老版本会被忽略 这个东西的作用相当于为通知分类 给用户选择通知消息的种类
+    val channelId =
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+            val channelId = "RustDeskForeground"
+            val channelName = "RustDesk屏幕分享服务状态"
+            val channel = NotificationChannel(
+                channelId,
+                channelName, NotificationManager.IMPORTANCE_DEFAULT
+            ).apply {
+                description = "Share your Android Screen with RustDeskService"
+            }
+            channel.lightColor = Color.BLUE
+            channel.lockscreenVisibility = Notification.VISIBILITY_PRIVATE
+            val service = ctx.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+            service.createNotificationChannel(channel)
+            channelId
+        } else {
+            ""
+        }
+
+    val notification: Notification = NotificationCompat.Builder(ctx, channelId)
+        .setOngoing(true)
+        .setPriority(NotificationCompat.PRIORITY_LOW)
+        .build()
+    ctx.startForeground(11, notification)
+}
+
+fun createNormalNotification(ctx: Context,title:String,text:String,type:String): Notification {
+    val channelId =
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+            val channelId = "RustDeskNormal"
+            val channelName = "RustDesk通知消息"
+            val channel = NotificationChannel(
+                channelId,
+                channelName, NotificationManager.IMPORTANCE_DEFAULT
+            ).apply {
+                description = "Share your Android Screen with RustDeskService"
+            }
+            channel.lightColor = Color.BLUE
+            channel.lockscreenVisibility = Notification.VISIBILITY_PUBLIC
+            val service = ctx.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+            service.createNotificationChannel(channel)
+            channelId
+        } else {
+            ""
+        }
+    val intent = Intent(ctx,MainActivity::class.java).apply {
+        flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED
+        action =Intent.ACTION_MAIN // 不设置会造成每次都重新启动一个新的Activity
+        addCategory(Intent.CATEGORY_LAUNCHER)
+        putExtra("type",type)
+    }
+    val pendingIntent = PendingIntent.getActivity(ctx,0,intent,FLAG_UPDATE_CURRENT)
+    return NotificationCompat.Builder(ctx, channelId)
+        .setSmallIcon(R.mipmap.ic_launcher)
+        .setContentTitle(title)
+        .setPriority(NotificationCompat.PRIORITY_HIGH) // 这里如果设置为低则不显示
+        .setContentText(text)
+        .setContentIntent(pendingIntent)
+        .setAutoCancel(true)
+        .build()
+}
+
+
+const val MY_PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE = 1
+
+fun checkPermissions(context: Context) {
+    val permissions: MutableList<String> = LinkedList()
+    addPermission(context,permissions, Manifest.permission.WRITE_EXTERNAL_STORAGE)
+    addPermission(context,permissions, Manifest.permission.RECORD_AUDIO)
+    addPermission(context,permissions, Manifest.permission.INTERNET)
+    addPermission(context,permissions, Manifest.permission.READ_PHONE_STATE)
+    if (permissions.isNotEmpty()) {
+        ActivityCompat.requestPermissions(
+            context as Activity, permissions.toTypedArray(),
+            MY_PERMISSIONS_REQUEST_READ_EXTERNAL_STORAGE
+        )
+    }
+}
+
+private fun addPermission(context:Context,permissionList: MutableList<String>, permission: String) {
+    if (ContextCompat.checkSelfPermission(
+            context,
+            permission
+        ) !== PackageManager.PERMISSION_GRANTED
+    ) {
+        permissionList.add(permission)
+    }
+}

android_doc.md

@@ -4,7 +4,7 @@
 
 ##### 原理 流程
 MediaProjectionManager -> MediaProjection
--> VirtualDisplay -> Surface -> MediaCodec
+-> VirtualDisplay -> Surface <- MediaCodec/ImageReader
 
 - 获取mediaProjectionResultIntent
     - **必须activity**
@@ -17,7 +17,7 @@ MediaProjectionManager -> MediaProjection
     - 通过后台服务获取MediaProjection
 
 - 创建Surface(理解为一个buf)和Surface消费者
-    - MediaCodec生成Surface传入VirtualDisplay的入参中
+    - MediaCodec(使用内置编码器)或者ImageReader(捕获原始数据)生成Surface传入VirtualDisplay的入参中
     - 设定编码等各类参数
 
 - 获取VirtualDisplay(Surface 生产者)
@@ -26,6 +26,13 @@ MediaProjectionManager -> MediaProjection
     - 创建VirtualDisplay的入参之一是Surface
     - 需要设定正确的VirtualDisplay尺寸
 
+#####方案A 捕获原始数据传入rust进行编码
+- 构建ImageReader生成Surface
+- **注意**：安卓捕获到的数据是RGBA格式，暂无BRGA的输出，在rust端需要调用libyuv中相应的rgbatoi420方法
+- 捕获到的数据存入一个bytearray，等待rust端调用获取
+
+#####方案B 捕获原始数据传入rust进行编码 !等待完善！
+- **自带的编码器无法直接控制流量，默认情况输出的帧率比较高，会造成网络堵塞延迟**
 - 获取编码后的buf
     - 通过MediaCodec回调获取到可用的数据
 - 通过jni传入Rust服务
@@ -56,12 +63,11 @@ MediaProjectionManager -> MediaProjection
   https://stackoverflow.com/questions/47531742/startforeground-fail-after-upgrade-to-android-8-1
   https://developer.android.com/reference/android/support/v4/app/NotificationCompat.Builder.html#NotificationCompat.Builder(android.content.Context)
 
-// TODO 使用 NotificationCompat 的区别
-
 <hr>
 
 ### 2.获取控制
-暂时可行的方案是使用安卓无障碍服务 参考droidVNC项目，但droidVNC的实现并不完善，droidVNC没有实现连续触控。
+暂时可行的方案是使用安卓无障碍服务 参考droidVNC项目，
+**目前暂无可用连续输入的方案，暂时只能做到控制端鼠标滑动抬起鼠标后才能发送这组控制到安卓端**
 
 #### 无障碍服务获取权限
 - https://developer.android.com/guide/topics/ui/accessibility/service?hl=zh-cn#manifest
@@ -93,24 +99,40 @@ MediaProjectionManager -> MediaProjection
     android:canPerformGestures="true" // 这里最关键
   />
   ```
-- 连续手势 https://developer.android.com/guide/topics/ui/accessibility/service?hl=zh-cn#continued-gestures
+- ~~连续手势 https://developer.android.com/guide/topics/ui/accessibility/service?hl=zh-cn#continued-gestures~~
+
+#### knox.remotecontrol 三星手机 专用控制方式
+
+https://docs.samsungknox.com/devref/knox-sdk/reference/com/samsung/android/knox/remotecontrol/package-summary.html
+
 
 <hr>
 
 ### 3.获取音频输入
 https://developer.android.google.cn/guide/topics/media/playback-capture?hl=zh-cn
 
-目前谷歌只开放了Android10系统同步音频内录功能
+**仅安卓10或更高可用**
+目前谷歌只开放了Android10及以上系统同步音频内录功能
 10之前录音的时候会截取原本系统的音频输出
 即 开启内录时候无法在手机上正常使用耳机扬声器输出
+且普通应用的声音默认不会被捕获
+
+**安卓10音频输入原理**
+- 音频权限相当于是MediaProjection的附属产物
+- 只有在成功获取MediaProjection，开启了ForegroundService才能使用
+- 相比于AudioRecord普通用法使用，将setAudioSource改为setAudioPlaybackCaptureConfig，这里的AudioPlaybackCaptureConfiguration的构建需要使用到之前成功获取的MediaProjection
+<br>
+- **一些注意事项**
+  - 使用AudioFormat.ENCODING_PCM_FLOAT，数值范围[-1,1]的32位浮点数据，对应了rust端opus编码器的输入格式。
+  - libopus库中使用的opus_encode_float，对于输入的音频数据长度有一定要求，安卓端输出的包过大需要分批发送
+    - https://stackoverflow.com/questions/46786922/how-to-confirm-opus-encode-buffer-size
+    - https://docs.rs/audiopus_sys/0.2.2/audiopus_sys/fn.opus_encode_float.html
+    - > For example, at 48 kHz the permitted values are 120, 240, 480, 960, 1920, and 2880. 
+  - 安卓11自带了opus输出，几年后或许可用
+
 
 <hr>
 
-### 其他
-- Kotlin 与 compose 版本设置问题
-    - https://stackoverflow.com/questions/67600344/jetpack-compose-on-kotlin-1-5-0
-    - 在根目录的gradle中 设置两个正确对应版本
-
 ### Rust JVM 互相调用
 
 rust端 引入 jni crate
@@ -137,37 +159,106 @@ pub unsafe extern "system" fn Java_com_carriez_flutter_1hbb_MainActivity_init(
 
 }
 ```
-- 注意，原项目包名flutter_hbb 带有下划线，通过安卓的编译提示获得的命名方式为如上。
+- 注意，原项目包名flutter_hbb 带有下划线，通过安卓的编译提示获得的命名方式为如上的`..._1hbb...`。
-
-- 使用jni的时候会出现无输出崩溃的情况
 
+- 使用jni的时候，如果不捕捉错误会出现无输出崩溃的情况
 - 将安卓的对象实例（Context）在init的过程中传入rust端，  
 context通过env.new_global_ref()变成全局引用
 env.get_java_vm()获取到jvm
   - 原理上 Rust端通过类找静态方法也可行，但在kotlin端测试失败，会遇到类名找不到，类静态方法找不到等问题，目前仅使用绑定具体context对象即可。
 - 将jvm和context 固定到全局变量中等待需要时候引用
-
 - 使用时，需要确保jvm与当前的线程绑定
 jvm.attach_current_thread_permanently()
-
 - 然后通过jvm获得env
 jvm.get_env()
-
 - 通过env.call_method()方法传入context.as_obj()使用对象的方法
+- ByteArray(Kotlin) 
+  - 会在JVM中编译成为java的byte[]
+  - rust端通过jbytearray接收，通过jni.rs的方法
+env.convert_byte_array()即可转化为Vec数组
 
-传递数据
+- FloatArray(Kotlin)
-Kotlin 中的 ByteArray 类 会在JVM中编译成为java的byte[]
+  - JAVA中的float[]
-byte[]通过jni传递到rust端时
+  - jni.rs中使用get_float_array_region方法写入到新buf中
-通过jni.rs的方法
-env.convert_byte_array()即可转化为Vec<u8>
 
-- 关于 sig
+- ByteBuffer(Kotlin/Java)
-  - (I)V ： 输入为Int 输出为Void
+  - 
-  - (I)Ljava/nio/ByteBuffer ： 输入为Int 输出为ByteBuffer  对应jni JByteBuffer 且env 有对应解析方法
+- 关于 sig 一些用例 https://imzy.vip/posts/55704/
-  - ()[B : 输入为空 输出为byte[](java) == ByteArray(kotlin) == jbytearray(jni env 中有对应解析方法到Vec\<u8\>)
+  - (I)V ： input:Int,output:Void
-  - 使用JValue枚举定义java变量
+  - (I)Ljava/nio/ByteBuffer ： input:Int,output:ByteBuffer. convert method:env.get_direct_buffer_address()
+  - ()[B : input:void,output:byte[](java) == ByteArray(kotlin) == jbytearray(jni env 中有对应解析方法到Vec\<u8\>)
+  - call java的方法时，使用JValue枚举定义java变量
 
-# BIG TODO
+
-音频
+<hr>
-连续输入
+
-服务关闭操作
+### UI交互相关
+收到无密码登录请求时，1.通知flutterUI页面问询用户2.通知栏通知用户
+- 否
+  - 返回给rust端否 java端不做处理
+- 是
+  - 返回给rust端是 java端开始采集音视频
+
+收到有密码登录请求，rust端可以自动判断
+- 否
+  - rust端自动处理返回密码错误
+- 是
+  - 通知java端 java端开始采集 同时在通知推送栏中推送消息
+
+#### 服务开启与关闭
+1.start listen
+安卓端用户手动开始服务监听 开启service
+获取视频权限 成功后 通知flutter将图标状态转为已开启 开启rust端的start_all()
+然后就可以被其他人请求连接
+
+2.login request
+验证成功的请求，
+安卓端开启视频 音频 输入的采集
+通知rust端logon response
+
+3.client close
+rust端会自动结束
+rust端发送结束指令给安卓端
+安卓端停止各项采集 但服务依然开启
+
+4.server close
+4-1
+close conn 
+用户点击断开连接
+安卓端停止各项采集
+发送close指令给rust让rust关闭这个conn
+
+4-1
+close totally 
+如果当前有连接则问是否断开
+是则先执行一遍4-1
+然后关闭整个service
+
+
+服务端主动关闭服务
+Config::set_option("stop_service","Y")
+服务端再次启动服务
+Config::set_option("stop_service","")
+
+
+### TODO 
+完善CM 当前连接的状态 控制音频和输入等开关 断开连接等功能
+横屏模式
+首次登录不显示id密码
+安卓前后分离的问题 通过IPC或者广播解耦
+
+<hr>
+
+### 其他
+- Kotlin 与 compose 版本设置问题
+    - https://stackoverflow.com/questions/67600344/jetpack-compose-on-kotlin-1-5-0
+    - 在根目录的gradle中 设置两个正确对应版本
+- 如果开发环境中安装了超过一种NDK版本，则会需要在app的build.gradle中指定NDK版本
+  ```
+  // build.gradle in app
+  android {
+    ...
+    compileSdkVersion 30
+    ndkVersion '22.1.7171670' 
+    ...
+  ```

lib/common.dart

@@ -1,8 +1,10 @@
 import 'package:flutter/material.dart';
 import 'dart:async';
 import 'package:flutter_easyloading/flutter_easyloading.dart';
+import 'package:flutter_hbb/server_page.dart';
 import 'package:tuple/tuple.dart';
 import 'dart:io';
+import 'main.dart';
 
 typedef F = String Function(String);
 
@@ -12,6 +14,7 @@ class Translator {
 
 class MyTheme {
   MyTheme._();
+
   static const Color grayBg = Color(0xFFEEEEEE);
   static const Color white = Color(0xFFFFFFFF);
   static const Color accent = Color(0xFF0071FF);
@@ -30,6 +33,7 @@ final ButtonStyle flatButtonStyle = TextButton.styleFrom(
 );
 
 void Function() loadingCancelCallback = null;
+
 void showLoading(String text, BuildContext context) {
   if (_hasDialog && context != null) {
     Navigator.pop(context);
@@ -71,6 +75,7 @@ void dismissLoading() {
 }
 
 bool _hasDialog = false;
+
 typedef BuildAlertDailog = Tuple3<Widget, Widget, List<Widget>> Function(
     void Function(void Function()));
 
@@ -107,9 +112,10 @@ void msgbox(String type, String title, String text, BuildContext context,
     [bool hasCancel]) {
   var wrap = (String text, void Function() onPressed) => ButtonTheme(
       padding: EdgeInsets.symmetric(horizontal: 20, vertical: 10),
-      materialTapTargetSize: MaterialTapTargetSize
+      materialTapTargetSize: MaterialTapTargetSize.shrinkWrap,
-          .shrinkWrap, //limits the touch area to the button area
+      //limits the touch area to the button area
-      minWidth: 0, //wraps child's width
+      minWidth: 0,
+      //wraps child's width
       height: 0,
       child: TextButton(
           style: flatButtonStyle,
@@ -165,12 +171,14 @@ class PasswordWidget extends StatefulWidget {
 
 class _PasswordWidgetState extends State<PasswordWidget> {
   bool _passwordVisible = false;
+
   @override
   Widget build(BuildContext context) {
     return TextField(
       autofocus: true,
       controller: widget.controller,
-      obscureText: !_passwordVisible, //This will obscure text dynamically
+      obscureText: !_passwordVisible,
+      //This will obscure text dynamically
       keyboardType: TextInputType.visiblePassword,
       decoration: InputDecoration(
         labelText: Translator.call('Password'),
@@ -202,3 +210,46 @@ Color str2color(String str, [alpha = 0xFF]) {
   hash = hash % 16777216;
   return Color((hash & 0xFF7FFF) | (alpha << 24));
 }
+
+toAndroidChannelInit() {
+  toAndroidChannel.setMethodCallHandler((call) async {
+    debugPrint("flutter got android msg");
+
+    try {
+      switch (call.method) {
+        case "try_start_without_auth":
+          {
+            var peerID = call.arguments["peerID"] as String;
+            var name = call.arguments["name"] as String;
+            ServerPage.serverModel.setPeer(false, name: name, id: peerID);
+            showLoginReqAlert(nowCtx, peerID, name);
+            debugPrint("from jvm:try_start_without_auth done");
+            break;
+          }
+        case "start_capture":
+          {
+            var peerID = call.arguments["peerID"] as String;
+            var name = call.arguments["name"] as String;
+            ServerPage.serverModel.setPeer(true, name: name, id: peerID);
+            break;
+          }
+        case "stop_capture":
+          {
+            ServerPage.serverModel.setPeer(false);
+            break;
+          }
+        case "on_permission_changed":
+          {
+            var name = call.arguments["name"] as String;
+            var value = call.arguments["value"] as String == "true";
+            debugPrint("from jvm:on_permission_changed,$name:$value");
+            ServerPage.serverModel.changeStatue(name, value);
+            break;
+          }
+      }
+    } catch (e) {
+      debugPrint("MethodCallHandler err:$e");
+    }
+    return null;
+  });
+}

lib/home_page.dart

@@ -1,11 +1,11 @@
 import 'package:flutter/material.dart';
-import 'package:flutter/services.dart';
 import 'package:provider/provider.dart';
 import 'package:tuple/tuple.dart';
 import 'package:package_info/package_info.dart';
 import 'package:url_launcher/url_launcher.dart';
 import 'dart:async';
 import 'common.dart';
+import 'main.dart';
 import 'model.dart';
 import 'remote_page.dart';
 import 'dart:io';
@@ -26,6 +26,7 @@ class _HomePageState extends State<HomePage> {
   @override
   void initState() {
     super.initState();
+    nowCtx = context;
     if (Platform.isAndroid) {
       Timer(Duration(seconds: 5), () {
         _updateUrl = FFI.getByName('software_update_url');
@@ -55,10 +56,11 @@ class _HomePageState extends State<HomePage> {
                         PopupMenuItem<String>(
                             child: Text(translate('ID Server')),
                             value: 'id_server'),
-                        PopupMenuItem<String>(
+                        Platform.isAndroid
-                            child: Text(translate('Share My Screen')),
+                            ? PopupMenuItem<String>(
-                            value: 'server'),
+                                child: Text(translate('Share My Screen')),
-                        // TODO only android
+                                value: 'server')
+                            : null,
                         PopupMenuItem<String>(
                             child: Text(translate('About') + ' RustDesk'),
                             value: 'about'),

lib/main.dart

@@ -1,19 +1,26 @@
 import 'package:flutter/material.dart';
+import 'package:flutter/services.dart';
 import 'package:flutter_hbb/server_page.dart';
 import 'package:provider/provider.dart';
 import 'package:firebase_analytics/firebase_analytics.dart';
 import 'package:firebase_analytics/observer.dart';
 import 'package:firebase_core/firebase_core.dart';
+import 'common.dart';
 import 'model.dart';
 import 'home_page.dart';
 
+const toAndroidChannel = MethodChannel("mChannel");
+BuildContext nowCtx;
+
 Future<Null> main() async {
   WidgetsFlutterBinding.ensureInitialized();
   await Firebase.initializeApp();
+  toAndroidChannelInit();
   runApp(App());
 }
 
 class App extends StatelessWidget {
+
   @override
   Widget build(BuildContext context) {
     final analytics = FirebaseAnalytics();

lib/model.dart

@@ -37,10 +37,15 @@ class FfiModel with ChangeNotifier {
   bool _direct;
 
   get permissions => _permissions;
+
   get initialized => _initialized;
+
   get display => _display;
+
   get secure => _secure;
+
   get direct => _direct;
+
   get pi => _pi;
 
   FfiModel() {
@@ -245,7 +250,9 @@ class CanvasModel with ChangeNotifier {
   }
 
   double get x => _x;
+
   double get y => _y;
+
   double get scale => _scale;
 
   void update(double x, double y, double scale) {
@@ -295,7 +302,7 @@ class CanvasModel with ChangeNotifier {
     notifyListeners();
   }
 
-  void clear([bool notify=false]) {
+  void clear([bool notify = false]) {
     _x = 0;
     _y = 0;
     _scale = 1.0;
@@ -314,10 +321,15 @@ class CursorModel with ChangeNotifier {
   double _displayOriginY = 0;
 
   ui.Image get image => _image;
+
   double get x => _x - _displayOriginX;
+
   double get y => _y - _displayOriginY;
+
   Offset get offset => Offset(_x, _y);
+
   double get hotx => _hotx;
+
   double get hoty => _hoty;
 
   // remote physical display coordinate
@@ -506,6 +518,61 @@ class CursorModel with ChangeNotifier {
   }
 }
 
+class ServerModel with ChangeNotifier {
+  bool _mediaOk;
+  bool _inputOk;
+
+  bool _peerEnabled;
+  String _peerName;
+  String _peerID;
+
+  bool get mediaOk => _mediaOk;
+
+  bool get inputOk => _inputOk;
+
+  bool get peerEnabled => _peerEnabled;
+
+  String get peerName => _peerName;
+
+  String get peerID => _peerID;
+
+  ServerModel() {
+    _mediaOk = false;
+    _inputOk = false;
+    _peerEnabled = false;
+    _peerName = "";
+    _peerID = "";
+  }
+
+  changeStatue(String name, bool value) {
+    switch (name) {
+      case "media":
+        _mediaOk = value;
+        break;
+      case "input":
+        _inputOk = value;
+        break;
+      default:
+        return;
+    }
+    notifyListeners();
+  }
+
+  setPeer(bool enabled, {String name = "", String id = ""}) {
+    _peerEnabled = enabled;
+    if (name != "") _peerName = name;
+    if (id != "") _peerID = id;
+    notifyListeners();
+  }
+
+  clearPeer() {
+    _peerEnabled = false;
+    _peerName = "";
+    _peerID = "";
+    notifyListeners();
+  }
+}
+
 class FFI {
   static String id = "";
   static String _dir = '';

lib/server_page.dart

@@ -1,34 +1,57 @@
+import 'dart:async';
 import 'package:flutter/material.dart';
-import 'package:flutter/services.dart';
 import 'package:flutter_hbb/model.dart';
+import 'package:provider/provider.dart';
 
 import 'common.dart';
+import 'main.dart';
 
-class ServerPage extends StatefulWidget {
+class ServerPage extends StatelessWidget {
-  @override
+  static final serverModel = ServerModel();
-  _ServerPageState createState() => _ServerPageState();
-}
 
-class _ServerPageState extends State<ServerPage> {
   @override
   Widget build(BuildContext context) {
-    return Scaffold(
+    // TODO: implement build
-        backgroundColor: MyTheme.grayBg,
+    return ChangeNotifierProvider.value(
-        appBar: AppBar(
+        value: serverModel,
-          centerTitle: true,
+        child: Scaffold(
-          title: const Text("Share My Screen"),
+            backgroundColor: MyTheme.grayBg,
-        ),
+            appBar: AppBar(
-        body: SingleChildScrollView(
+              centerTitle: true,
-          child: Center(
+              title: const Text("Share My Screen"),
-            child: Column(
+              actions: [
-              mainAxisAlignment: MainAxisAlignment.start,
+                PopupMenuButton<String>(
-              children: [
+                    itemBuilder: (context) {
-                ServerInfo(),
+                      return [
-                PermissionChecker(),
+                        PopupMenuItem(
+                          child: Text("修改服务ID"),
+                          value: "changeID",
+                          enabled: false,
+                        ),
+                        PopupMenuItem(
+                          child: Text("修改服务密码"),
+                          value: "changeID",
+                          enabled: false,
+                        )
+                      ];
+                    },
+                    onSelected: (value) =>
+                        debugPrint("PopupMenuItem onSelected:$value"))
               ],
             ),
-          ),
+            body: SingleChildScrollView(
-        ));
+              child: Center(
+                child: Column(
+                  mainAxisAlignment: MainAxisAlignment.start,
+                  children: [
+                    ServerInfo(),
+                    PermissionChecker(),
+                    ConnectionManager(),
+                    SizedBox.fromSize(size: Size(0, 15.0)), // Bottom padding
+                  ],
+                ),
+              ),
+            )));
   }
 }
 
@@ -105,60 +128,62 @@ class PermissionChecker extends StatefulWidget {
 }
 
 class _PermissionCheckerState extends State<PermissionChecker> {
-  static const toAndroidChannel = MethodChannel("mChannel");
+  @override
-
+  void initState() {
-  var videoOk = false;
+    super.initState();
-  var inputOk = false;
+    nowCtx = context;
-  var audioOk = false;
+  }
 
   @override
   Widget build(BuildContext context) {
+    final serverModel = Provider.of<ServerModel>(context);
+
     return myCard(Column(
       crossAxisAlignment: CrossAxisAlignment.start,
       children: [
         cardTitle("权限列表"),
-        PermissionRow("视频权限", videoOk, _toAndroidGetPer),
+        PermissionRow("媒体权限", serverModel.mediaOk, _toAndroidInitService),
         const Divider(height: 0),
-        PermissionRow("音频权限", videoOk, () => {debugPrint("获取视频权限")}),
+        PermissionRow("输入权限", serverModel.inputOk, _toAndroidCheckInput),
-        const Divider(height: 0),
-        PermissionRow("输入权限", inputOk, _toAndroidCheckInput),
         const Divider(),
-        Row(
+        serverModel.mediaOk
-          mainAxisAlignment: MainAxisAlignment.spaceAround,
+            ? ElevatedButton.icon(
-          children: [
-            TextButton.icon(
-                icon: Icon(Icons.play_arrow),
-                onPressed: _toAndroidStartSer,
-                label: Text("Start")),
-            TextButton.icon(
                 icon: Icon(Icons.stop),
-                onPressed: _toAndroidStopSer,
+                onPressed: _toAndroidStopService,
-                label: Text("Stop")),
+                label: Text("Stop"))
-          ],
+            : ElevatedButton.icon(
-        )
+                icon: Icon(Icons.play_arrow),
+                onPressed: _toAndroidInitService,
+                label: Text("Start")),
       ],
     ));
   }
+}
 
-  Future<Null> _toAndroidGetPer() async {
+void showLoginReqAlert(BuildContext context, String peerID, String name) {
-    bool res = await toAndroidChannel.invokeMethod("getPer");
+  debugPrint("got try_start_without_auth");
-    debugPrint("_toAndroidGetPer:$res");
+  showDialog(
-  }
+      context: context,
-
+      builder: (context) => AlertDialog(
-  Future<Null> _toAndroidStartSer() async {
+            title: Text("收到连接请求"),
-    bool res = await toAndroidChannel.invokeMethod("startSer");
+            content: Text("是否同意来自$name:$peerID的控制？"),
-    debugPrint("_toAndroidStartSer:$res");
+            actions: [
-  }
+              TextButton(
-
+                  child: Text("接受"),
-  Future<Null> _toAndroidStopSer() async {
+                  onPressed: () {
-    bool res = await toAndroidChannel.invokeMethod("stopSer");
+                    FFI.setByName("login_res", "true");
-    debugPrint("_toAndroidStopSer:$res");
+                    _toAndroidStartCapture();
-  }
+                    ServerPage.serverModel.setPeer(true);
-
+                    Navigator.of(context).pop();
-  Future<Null> _toAndroidCheckInput() async {
+                  }),
-    bool res = await toAndroidChannel.invokeMethod("checkInput");
+              TextButton(
-    debugPrint("_toAndroidStopSer:$res");
+                  child: Text("不接受"),
-  }
+                  onPressed: () {
+                    FFI.setByName("login_res", "false");
+                    Navigator.of(context).pop();
+                  })
+            ],
+          ));
 }
 
 class PermissionRow extends StatelessWidget {
@@ -183,7 +208,7 @@ class PermissionRow extends StatelessWidget {
                   fontSize: 16.0, color: isOk ? Colors.green : Colors.grey)),
         ])),
         TextButton(
-            onPressed: onPressed,
+            onPressed: isOk ? null : onPressed,
             child: const Text(
               "去开启",
               style: TextStyle(fontWeight: FontWeight.bold),
@@ -193,6 +218,37 @@ class PermissionRow extends StatelessWidget {
   }
 }
 
+class ConnectionManager extends StatelessWidget {
+  @override
+  Widget build(BuildContext context) {
+    final serverModel = Provider.of<ServerModel>(context);
+    var info =
+        "${serverModel.peerName != "" ? serverModel.peerName : "NA"}-${serverModel.peerID != "" ? serverModel.peerID : "NA"}";
+    return serverModel.peerEnabled
+        ? myCard(Column(
+            crossAxisAlignment: CrossAxisAlignment.start,
+            children: [
+              cardTitle("当前连接"),
+              Padding(
+                padding: EdgeInsets.symmetric(vertical: 5.0),
+                child: Text(info, style: TextStyle(color: Colors.grey)),
+              ),
+              ElevatedButton.icon(
+                  style: ButtonStyle(
+                      backgroundColor: MaterialStateProperty.all(Colors.red)),
+                  icon: Icon(Icons.close),
+                  onPressed: () {
+                    FFI.setByName("close_conn");
+                    _toAndroidStopCapture();
+                    serverModel.setPeer(false);
+                  },
+                  label: Text("断开连接"))
+            ],
+          ))
+        : SizedBox.shrink();
+  }
+}
+
 Widget cardTitle(String text) {
   return Padding(
       padding: EdgeInsets.symmetric(vertical: 5.0),
@@ -201,18 +257,47 @@ Widget cardTitle(String text) {
         style: TextStyle(
           fontFamily: 'WorkSans',
           fontWeight: FontWeight.bold,
-          fontSize: 25,
+          fontSize: 22,
           color: MyTheme.accent80,
         ),
       ));
 }
 
 Widget myCard(Widget child) {
-  return Card(
+  return Container(
-    margin: EdgeInsets.all(15.0),
+      width: double.maxFinite,
-    child: Padding(
+      child: Card(
-      padding: EdgeInsets.symmetric(vertical: 15.0, horizontal: 30.0),
+        margin: EdgeInsets.fromLTRB(15.0, 15.0, 15.0, 0),
-      child: child,
+        child: Padding(
-    ),
+          padding: EdgeInsets.symmetric(vertical: 15.0, horizontal: 30.0),
-  );
+          child: child,
+        ),
+      ));
+}
+
+Future<Null> _toAndroidInitService() async {
+  bool res = await toAndroidChannel.invokeMethod("init_service");
+  FFI.setByName("start_service");
+  debugPrint("_toAndroidInitService:$res");
+}
+
+Future<Null> _toAndroidStartCapture() async {
+  bool res = await toAndroidChannel.invokeMethod("start_capture");
+  debugPrint("_toAndroidStartCapture:$res");
+}
+
+Future<Null> _toAndroidStopCapture() async {
+  bool res = await toAndroidChannel.invokeMethod("stop_capture");
+  debugPrint("_toAndroidStopCapture:$res");
+}
+
+Future<Null> _toAndroidStopService() async {
+  FFI.setByName("stop_service");
+  bool res = await toAndroidChannel.invokeMethod("stop_service");
+  debugPrint("_toAndroidStopSer:$res");
+}
+
+Future<Null> _toAndroidCheckInput() async {
+  bool res = await toAndroidChannel.invokeMethod("check_input");
+  debugPrint("_toAndroidStopSer:$res");
 }