konevsa/rustdesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

codec set quality seperately and refactor network delay

Browse Source 
Signed-off-by: 21pages <pages21@163.com>
This commit is contained in:
21pages 2023-07-19 13:11:24 +08:00
parent 633c80d5e4
commit 2133f91089
9 changed files with 457 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
libs/scrap/examples/benchmark.rs
View File

@ -2,7 +2,7 @@ use docopt::Docopt;
use hbb_common::env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV};
use scrap::{
aom::{AomDecoder, AomDecoderConfig, AomEncoder, AomEncoderConfig},
codec::{EncoderApi, EncoderCfg},
codec::{EncoderApi, EncoderCfg, Quality as Q},
Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig,
VpxVideoCodecId::{self, *},
STRIDE_ALIGN,
@ -15,13 +15,14 @@ const USAGE: &'static str = "
Codec benchmark.
Usage:
benchmark [--count=COUNT] [--bitrate=KBS] [--hw-pixfmt=PIXFMT]
benchmark [--count=COUNT] [--quality=QUALITY] [--hw-pixfmt=PIXFMT]
benchmark (-h | --help)
Options:
-h --help Show this screen.
--count=COUNT Capture frame count [default: 100].
--bitrate=KBS Video bitrate in kilobits per second [default: 5000].
--quality=QUALITY Video quality [default: Balanced].
Valid values: Best, Balanced, Low.
--hw-pixfmt=PIXFMT Hardware codec pixfmt. [default: i420]
Valid values: i420, nv12.
";
@ -29,7 +30,7 @@ Options:
#[derive(Debug, serde::Deserialize)]
struct Args {
flag_count: usize,
flag_bitrate: usize,
flag_quality: Quality,
flag_hw_pixfmt: Pixfmt,
}
@ -39,20 +40,32 @@ enum Pixfmt {
NV12,
}
#[derive(Debug, serde::Deserialize)]
enum Quality {
Best,
Balanced,
Low,
}
fn main() {
init_from_env(Env::default().filter_or(DEFAULT_FILTER_ENV, "info"));
let args: Args = Docopt::new(USAGE)
.and_then(|d| d.deserialize())
.unwrap_or_else(|e| e.exit());
let bitrate_k = args.flag_bitrate;
let quality = args.flag_quality;
let yuv_count = args.flag_count;
let (yuvs, width, height) = capture_yuv(yuv_count);
println!(
"benchmark {}x{} bitrate:{}k hw_pixfmt:{:?}",
width, height, bitrate_k, args.flag_hw_pixfmt
"benchmark {}x{} quality:{:?}k hw_pixfmt:{:?}",
width, height, quality, args.flag_hw_pixfmt
);
[VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, bitrate_k, yuv_count));
test_av1(&yuvs, width, height, bitrate_k, yuv_count);
let quality = match quality {
Quality::Best => Q::Best,
Quality::Balanced => Q::Balanced,
Quality::Low => Q::Low,
};
[VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, quality, yuv_count));
test_av1(&yuvs, width, height, quality, yuv_count);
#[cfg(feature = "hwcodec")]
{
use hwcodec::AVPixelFormat;
@ -61,7 +74,7 @@ fn main() {
Pixfmt::NV12 => AVPixelFormat::AV_PIX_FMT_NV12,
};
let yuvs = hw::vpx_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt);
hw::test(&yuvs, width, height, bitrate_k, yuv_count, hw_pixfmt);
hw::test(&yuvs, width, height, quality, yuv_count, hw_pixfmt);
}
}
@ -95,14 +108,14 @@ fn test_vpx(
yuvs: &Vec<Vec<u8>>,
width: usize,
height: usize,
bitrate_k: usize,
quality: Q,
yuv_count: usize,
) {
let config = EncoderCfg::VPX(VpxEncoderConfig {
width: width as _,
height: height as _,
timebase: [1, 1000],
bitrate: bitrate_k as _,
quality,
codec: codec_id,
num_threads: (num_cpus::get() / 2) as _,
});
@ -148,11 +161,11 @@ fn test_vpx(
);
}
fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, bitrate_k: usize, yuv_count: usize) {
fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, quality: Q, yuv_count: usize) {
let config = EncoderCfg::AOM(AomEncoderConfig {
width: width as _,
height: height as _,
bitrate: bitrate_k as _,
quality,
});
let mut encoder = AomEncoder::new(config).unwrap();
let start = Instant::now();
@ -208,17 +221,18 @@ mod hw {
yuvs: &Vec<Vec<u8>>,
width: usize,
height: usize,
bitrate_k: usize,
quality: Q,
yuv_count: usize,
pixfmt: AVPixelFormat,
) {
let bitrate = scrap::hwcodec::HwEncoder::convert_quality(quality);
let ctx = EncodeContext {
name: String::from(""),
width: width as _,
height: height as _,
pixfmt,
align: 0,
bitrate: (bitrate_k * 1000) as _,
bitrate: bitrate as i32 * 1000,
timebase: [1, 30],
gop: 60,
quality: Quality_Default,

36
libs/scrap/examples/record-screen.rs
View File

@ -13,7 +13,7 @@ use std::time::{Duration, Instant};
use std::{io, thread};
use docopt::Docopt;
use scrap::codec::{EncoderApi, EncoderCfg};
use scrap::codec::{EncoderApi, EncoderCfg, Quality as Q};
use webm::mux;
use webm::mux::Track;
@ -24,17 +24,18 @@ const USAGE: &'static str = "
Simple WebM screen capture.
Usage:
record-screen <path> [--time=<s>] [--fps=<fps>] [--bv=<kbps>] [--ba=<kbps>] [--codec CODEC]
record-screen <path> [--time=<s>] [--fps=<fps>] [--quality=<quality>] [--ba=<kbps>] [--codec CODEC]
record-screen (-h | --help)
Options:
-h --help Show this screen.
--time=<s> Recording duration in seconds.
--fps=<fps> Frames per second [default: 30].
--bv=<kbps> Video bitrate in kilobits per second [default: 5000].
--ba=<kbps> Audio bitrate in kilobits per second [default: 96].
--codec CODEC Configure the codec used. [default: vp9]
Valid values: vp8, vp9.
-h --help Show this screen.
--time=<s> Recording duration in seconds.
--fps=<fps> Frames per second [default: 30].
--quality=<quality> Video quality [default: Balanced].
Valid values: Best, Balanced, Low.
--ba=<kbps> Audio bitrate in kilobits per second [default: 96].
--codec CODEC Configure the codec used. [default: vp9]
Valid values: vp8, vp9.
";
#[derive(Debug, serde::Deserialize)]
@ -43,7 +44,14 @@ struct Args {
flag_codec: Codec,
flag_time: Option<u64>,
flag_fps: u64,
flag_bv: u32,
flag_quality: Quality,
}
#[derive(Debug, serde::Deserialize)]
enum Quality {
Best,
Balanced,
Low,
}
#[derive(Debug, serde::Deserialize)]
@ -97,12 +105,16 @@ fn main() -> io::Result<()> {
let mut vt = webm.add_video_track(width, height, None, mux_codec);
// Setup the encoder.
let quality = match args.flag_quality {
Quality::Best => Q::Best,
Quality::Balanced => Q::Balanced,
Quality::Low => Q::Low,
};
let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig {
width,
height,
timebase: [1, 1000],
bitrate: args.flag_bv,
quality,
codec: vpx_codec,
num_threads: 0,
}))

79
libs/scrap/src/common/aom.rs
View File

@ -6,6 +6,7 @@
include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs"));
use crate::codec::{base_bitrate, Quality};
use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN};
use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result};
use hbb_common::{
@ -43,7 +44,7 @@ impl Default for aom_image_t {
pub struct AomEncoderConfig {
pub width: u32,
pub height: u32,
pub bitrate: u32,
pub quality: Quality,
}
pub struct AomEncoder {
@ -56,7 +57,6 @@ pub struct AomEncoder {
mod webrtc {
use super::*;
const kQpMin: u32 = 10;
const kUsageProfile: u32 = AOM_USAGE_REALTIME;
const kMinQindex: u32 = 145; // Min qindex threshold for QP scaling.
const kMaxQindex: u32 = 205; // Max qindex threshold for QP scaling.
@ -65,7 +65,8 @@ mod webrtc {
const kRtpTicksPerSecond: i32 = 90000;
const kMinimumFrameRate: f64 = 1.0;
const kQpMax: u32 = 25; // to-do: webrtc use dynamic value, no more than 63
pub const DEFAULT_Q_MAX: u32 = 56; // no more than 63
pub const DEFAULT_Q_MIN: u32 = 12; // no more than 63, litter than q_max
fn number_of_threads(width: u32, height: u32, number_of_cores: usize) -> u32 {
// Keep the number of encoder threads equal to the possible number of
@ -78,7 +79,7 @@ mod webrtc {
} else {
// Use 2 threads for low res on ARM.
#[cfg(any(target_arch = "arm", target_arch = "aarch64", target_os = "android"))]
if (width * height >= 320 * 180 && number_of_cores > 2) {
if width * height >= 320 * 180 && number_of_cores > 2 {
return 2;
}
// 1 thread less than VGA.
@ -122,11 +123,23 @@ mod webrtc {
c.g_threads = number_of_threads(cfg.width, cfg.height, num_cpus::get());
c.g_timebase.num = 1;
c.g_timebase.den = kRtpTicksPerSecond;
c.rc_target_bitrate = cfg.bitrate; // kilobits/sec.
c.g_input_bit_depth = kBitDepth;
c.kf_mode = aom_kf_mode::AOM_KF_DISABLED;
c.rc_min_quantizer = kQpMin;
c.rc_max_quantizer = kQpMax;
let (q_min, q_max, b) = AomEncoder::convert_quality(cfg.quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
} else {
c.rc_min_quantizer = DEFAULT_Q_MIN;
c.rc_max_quantizer = DEFAULT_Q_MAX;
}
let base_bitrate = base_bitrate(cfg.width as _, cfg.height as _);
let bitrate = base_bitrate * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
} else {
c.rc_target_bitrate = base_bitrate;
}
c.rc_undershoot_pct = 50;
c.rc_overshoot_pct = 50;
c.rc_buf_initial_sz = 600;
@ -259,11 +272,24 @@ impl EncoderApi for AomEncoder {
true
}
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
new_enc_cfg.rc_target_bitrate = bitrate;
call_aom!(aom_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
return Ok(());
fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
let mut c = unsafe { *self.ctx.config.enc.to_owned() };
let (q_min, q_max, b) = Self::convert_quality(quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
}
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
}
call_aom!(aom_codec_enc_config_set(&mut self.ctx, &c));
Ok(())
}
fn bitrate(&self) -> u32 {
let c = unsafe { *self.ctx.config.enc.to_owned() };
c.rc_target_bitrate
}
}
@ -319,6 +345,35 @@ impl AomEncoder {
..Default::default()
}
}
pub fn convert_quality(quality: Quality) -> (u32, u32, u32) {
// we can use lower bitrate for av1
match quality {
Quality::Best => (12, 25, 100),
Quality::Balanced => (12, 35, 100 * 2 / 3),
Quality::Low => (18, 45, 50),
Quality::Custom(b) => {
let (q_min, q_max) = Self::calc_q_values(b);
(q_min, q_max, b)
}
}
}
#[inline]
fn calc_q_values(b: u32) -> (u32, u32) {
let b = std::cmp::min(b, 200);
let q_min1: i32 = 24;
let q_min2 = 12;
let q_max1 = 45;
let q_max2 = 25;
let t = b as f32 / 200.0;
let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
(q_min, q_max)
}
}
impl Drop for AomEncoder {

36
libs/scrap/src/common/codec.rs
View File

@ -42,7 +42,7 @@ pub struct HwEncoderConfig {
pub name: String,
pub width: usize,
pub height: usize,
pub bitrate: i32,
pub quality: Quality,
}
#[derive(Debug, Clone)]
@ -61,7 +61,9 @@ pub trait EncoderApi {
fn use_yuv(&self) -> bool;
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>;
fn set_quality(&mut self, quality: Quality) -> ResultType<()>;
fn bitrate(&self) -> u32;
}
pub struct Encoder {
@ -471,3 +473,33 @@ fn enable_hwcodec_option() -> bool {
}
return true; // default is true
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Quality {
Best,
Balanced,
Low,
Custom(u32),
}
impl Default for Quality {
fn default() -> Self {
Self::Balanced
}
}
pub fn base_bitrate(width: u32, height: u32) -> u32 {
#[allow(unused_mut)]
let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9
if base_bitrate == 0 {
base_bitrate = 1920 * 1080 / 1000;
}
#[cfg(target_os = "android")]
{
// fix when android screen shrinks
let fix = crate::Display::fix_quality() as u32;
log::debug!("Android screen, fix quality:{}", fix);
base_bitrate = base_bitrate * fix;
}
base_bitrate
}

39
libs/scrap/src/common/hwcodec.rs
View File

@ -1,5 +1,5 @@
use crate::{
codec::{EncoderApi, EncoderCfg},
codec::{base_bitrate, EncoderApi, EncoderCfg},
hw, ImageFormat, ImageRgb, HW_STRIDE_ALIGN,
};
use hbb_common::{
@ -34,6 +34,9 @@ pub struct HwEncoder {
yuv: Vec<u8>,
pub format: DataFormat,
pub pixfmt: AVPixelFormat,
width: u32,
height: u32,
bitrate: u32, //kbs
}
impl EncoderApi for HwEncoder {
@ -43,13 +46,19 @@ impl EncoderApi for HwEncoder {
{
match cfg {
EncoderCfg::HW(config) => {
let b = Self::convert_quality(config.quality);
let base_bitrate = base_bitrate(config.width as _, config.height as _);
let mut bitrate = base_bitrate * b / 100;
if base_bitrate <= 0 {
bitrate = base_bitrate;
}
let ctx = EncodeContext {
name: config.name.clone(),
width: config.width as _,
height: config.height as _,
pixfmt: DEFAULT_PIXFMT,
align: HW_STRIDE_ALIGN as _,
bitrate: config.bitrate * 1000,
bitrate: bitrate as i32 * 1000,
timebase: DEFAULT_TIME_BASE,
gop: DEFAULT_GOP,
quality: DEFAULT_HW_QUALITY,
@ -70,6 +79,9 @@ impl EncoderApi for HwEncoder {
yuv: vec![],
format,
pixfmt: ctx.pixfmt,
width: ctx.width as _,
height: ctx.height as _,
bitrate,
}),
Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
}
@ -114,10 +126,19 @@ impl EncoderApi for HwEncoder {
false
}
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
self.encoder.set_bitrate((bitrate * 1000) as _).ok();
fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> {
let b = Self::convert_quality(quality);
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 {
self.encoder.set_bitrate((bitrate * 1000) as _).ok();
self.bitrate = bitrate;
}
Ok(())
}
fn bitrate(&self) -> u32 {
self.bitrate
}
}
impl HwEncoder {
@ -159,6 +180,16 @@ impl HwEncoder {
Err(_) => Ok(Vec::<EncodeFrame>::new()),
}
}
pub fn convert_quality(quality: crate::codec::Quality) -> u32 {
use crate::codec::Quality;
match quality {
Quality::Best => 150,
Quality::Balanced => 100,
Quality::Low => 50,
Quality::Custom(b) => b,
}
}
}
pub struct HwDecoder {

78
libs/scrap/src/common/vpxcodec.rs
View File

@ -7,7 +7,7 @@ use hbb_common::log;
use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame};
use hbb_common::ResultType;
use crate::codec::EncoderApi;
use crate::codec::{base_bitrate, EncoderApi, Quality};
use crate::{GoogleImage, STRIDE_ALIGN};
use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
@ -19,6 +19,9 @@ use std::{ptr, slice};
generate_call_macro!(call_vpx, false);
generate_call_ptr_macro!(call_vpx_ptr);
const DEFAULT_QP_MAX: u32 = 56; // no more than 63
const DEFAULT_QP_MIN: u32 = 12; // no more than 63
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum VpxVideoCodecId {
VP8,
@ -64,8 +67,9 @@ impl EncoderApi for VpxEncoder {
c.g_h = config.height;
c.g_timebase.num = config.timebase[0];
c.g_timebase.den = config.timebase[1];
c.rc_target_bitrate = config.bitrate;
c.rc_undershoot_pct = 95;
// When the data buffer falls below this percentage of fullness, a dropped frame is indicated. Set the threshold to zero (0) to disable this feature.
// In dynamic scenes, low bitrate gets low fps while high bitrate gets high fps.
c.rc_dropframe_thresh = 25;
c.g_threads = if config.num_threads == 0 {
num_cpus::get() as _
@ -79,6 +83,21 @@ impl EncoderApi for VpxEncoder {
// c.kf_min_dist = 0;
// c.kf_max_dist = 999999;
c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot
let (q_min, q_max, b) = Self::convert_quality(config.quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
} else {
c.rc_min_quantizer = DEFAULT_QP_MIN;
c.rc_max_quantizer = DEFAULT_QP_MAX;
}
let base_bitrate = base_bitrate(config.width as _, config.height as _);
let bitrate = base_bitrate * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
} else {
c.rc_target_bitrate = base_bitrate;
}
/*
The VPX encoder supports two-pass encoding for rate control purposes.
@ -177,11 +196,24 @@ impl EncoderApi for VpxEncoder {
true
}
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
new_enc_cfg.rc_target_bitrate = bitrate;
call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
return Ok(());
fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
let mut c = unsafe { *self.ctx.config.enc.to_owned() };
let (q_min, q_max, b) = Self::convert_quality(quality);
if q_min > 0 && q_min < q_max && q_max < 64 {
c.rc_min_quantizer = q_min;
c.rc_max_quantizer = q_max;
}
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 {
c.rc_target_bitrate = bitrate;
}
call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &c));
Ok(())
}
fn bitrate(&self) -> u32 {
let c = unsafe { *self.ctx.config.enc.to_owned() };
c.rc_target_bitrate
}
}
@ -258,6 +290,34 @@ impl VpxEncoder {
..Default::default()
}
}
fn convert_quality(quality: Quality) -> (u32, u32, u32) {
match quality {
Quality::Best => (6, 45, 150),
Quality::Balanced => (12, 56, 100 * 2 / 3),
Quality::Low => (18, 56, 50),
Quality::Custom(b) => {
let (q_min, q_max) = Self::calc_q_values(b);
(q_min, q_max, b)
}
}
}
#[inline]
fn calc_q_values(b: u32) -> (u32, u32) {
let b = std::cmp::min(b, 200);
let q_min1: i32 = 36;
let q_min2 = 12;
let q_max1 = 56;
let q_max2 = 37;
let t = b as f32 / 200.0;
let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
(q_min, q_max)
}
}
impl Drop for VpxEncoder {
@ -289,8 +349,8 @@ pub struct VpxEncoderConfig {
pub height: c_uint,
/// The timebase numerator and denominator (in seconds).
pub timebase: [c_int; 2],
/// The target bitrate (in kilobits per second).
pub bitrate: c_uint,
/// The image quality
pub quality: Quality,
/// The codec
pub codec: VpxVideoCodecId,
pub num_threads: u32,

15
src/server/connection.rs
View File

@ -196,6 +196,7 @@ pub struct Connection {
#[cfg(not(any(feature = "flatpak", feature = "appimage")))]
tx_desktop_ready: mpsc::Sender<()>,
closed: bool,
delay_response_instant: Instant,
}
impl ConnInner {
@ -326,6 +327,7 @@ impl Connection {
#[cfg(not(any(feature = "flatpak", feature = "appimage")))]
tx_desktop_ready: _tx_desktop_ready,
closed: false,
delay_response_instant: Instant::now(),
};
if !conn.on_open(addr).await {
conn.closed = true;
@ -593,18 +595,19 @@ impl Connection {
break;
}
let time = get_time();
let mut qos = video_service::VIDEO_QOS.lock().unwrap();
if time > 0 && conn.last_test_delay == 0 {
conn.last_test_delay = time;
let mut msg_out = Message::new();
let qos = video_service::VIDEO_QOS.lock().unwrap();
msg_out.set_test_delay(TestDelay{
time,
last_delay:conn.network_delay.unwrap_or_default(),
target_bitrate:qos.bitrate(),
target_bitrate: qos.bitrate(),
..Default::default()
});
conn.inner.send(msg_out.into());
}
qos.user_delay_response_elapsed(conn.inner.id(), conn.delay_response_instant.elapsed().as_millis());
}
}
}
@ -1188,7 +1191,9 @@ impl Connection {
#[inline]
#[cfg(not(any(target_os = "android", target_os = "ios")))]
fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) {
self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok();
self.tx_input
.send(MessageInput::Pointer((msg, conn_id)))
.ok();
}
#[inline]
@ -1553,6 +1558,7 @@ impl Connection {
.unwrap()
.user_network_delay(self.inner.id(), new_delay);
self.network_delay = Some(new_delay);
self.delay_response_instant = Instant::now();
}
} else if let Some(message::Union::SwitchSidesResponse(_s)) = msg.union {
#[cfg(feature = "flutter")]
@ -1590,7 +1596,8 @@ impl Connection {
self.input_mouse(me, self.inner.id());
}
}
Some(message::Union::PointerDeviceEvent(pde)) => {
Some(message::Union::PointerDeviceEvent(pde)) =>
{
#[cfg(not(any(target_os = "android", target_os = "ios")))]
if self.peer_keyboard_enabled() {
MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst);

277
src/server/video_qos.rs
View File

@ -1,4 +1,5 @@
use super::*;
use scrap::codec::Quality;
use std::time::Duration;
pub const FPS: u32 = 30;
pub const MIN_FPS: u32 = 1;
@ -18,21 +19,29 @@ impl Percent for ImageQuality {
}
}
#[derive(Default, Debug)]
#[derive(Default, Debug, Copy, Clone)]
struct Delay {
state: DelayState,
staging_state: DelayState,
delay: u32,
counter: u32,
slower_than_old_state: Option<bool>,
}
#[derive(Default, Debug, Copy, Clone)]
struct UserData {
full_speed_fps: Option<u32>,
custom_fps: Option<u32>,
quality: Option<(i32, i64)>, // (quality, time)
delay: Option<(DelayState, u32, usize)>, // (state, ms, counter)
quality: Option<(i64, Quality)>, // (time, quality)
delay: Option<Delay>,
response_delayed: bool,
}
pub struct VideoQoS {
width: u32,
height: u32,
fps: u32,
target_bitrate: u32,
updated: bool,
quality: Quality,
users: HashMap<i32, UserData>,
bitrate_store: u32,
}
#[derive(PartialEq, Debug, Clone, Copy)]
@ -43,6 +52,12 @@ enum DelayState {
Broken = 1000,
}
impl Default for DelayState {
fn default() -> Self {
DelayState::Normal
}
}
impl DelayState {
fn from_delay(delay: u32) -> Self {
if delay > DelayState::Broken as u32 {
@ -61,24 +76,19 @@ impl Default for VideoQoS {
fn default() -> Self {
VideoQoS {
fps: FPS,
width: 0,
height: 0,
target_bitrate: 0,
updated: false,
quality: Default::default(),
users: Default::default(),
bitrate_store: 0,
}
}
}
impl VideoQoS {
pub fn set_size(&mut self, width: u32, height: u32) {
if width == 0 || height == 0 {
return;
}
self.width = width;
self.height = height;
}
#[derive(Debug, PartialEq, Eq)]
pub enum RefreshType {
SetImageQuality,
}
impl VideoQoS {
pub fn spf(&self) -> Duration {
Duration::from_secs_f32(1. / (self.fps as f32))
}
@ -87,24 +97,23 @@ impl VideoQoS {
self.fps
}
pub fn bitrate(&self) -> u32 {
self.target_bitrate
pub fn store_bitrate(&mut self, bitrate: u32) {
self.bitrate_store = bitrate;
}
pub fn check_if_updated(&mut self) -> bool {
if self.updated {
self.updated = false;
return true;
}
return false;
pub fn bitrate(&self) -> u32 {
self.bitrate_store
}
pub fn quality(&self) -> Quality {
self.quality
}
pub fn abr_enabled() -> bool {
"N" != Config::get_option("enable-abr")
}
pub fn refresh(&mut self) {
let mut updated = false;
pub fn refresh(&mut self, typ: Option<RefreshType>) {
// fps
let user_fps = |u: &UserData| {
// full_speed_fps
@ -117,13 +126,19 @@ impl VideoQoS {
}
// delay
if let Some(delay) = u.delay {
fps = match delay.0 {
fps = match delay.state {
DelayState::Normal => fps,
DelayState::LowDelay => fps,
DelayState::HighDelay => fps / 2,
DelayState::Broken => fps / 4,
}
}
// delay response
if u.response_delayed {
if fps > MIN_FPS + 2 {
fps = MIN_FPS + 2;
}
}
return fps;
};
let mut fps = self
@ -136,64 +151,79 @@ impl VideoQoS {
if fps > MAX_FPS {
fps = MAX_FPS;
}
if fps != self.fps {
self.fps = fps;
updated = true;
}
self.fps = fps;
// quality
// latest image quality
let latest = self
let latest_quality = self
.users
.iter()
// .map(|(_, u)| u.quality)
.filter(|u| u.1.quality != None)
.max_by(|u1, u2| {
u1.1.quality
.unwrap_or_default()
.1
.cmp(&u2.1.quality.unwrap_or_default().1)
});
let quality = if let Some((id, data)) = latest {
let mut quality = data.quality.unwrap_or_default().0;
if quality <= 0 {
quality = ImageQuality::Balanced.as_percent() as _;
}
// use latest's delay for quality
if Self::abr_enabled() {
if let Some(Some((delay, _, _))) = self.users.get(id).map(|u| u.delay) {
quality = match delay {
DelayState::Normal => quality,
DelayState::LowDelay => std::cmp::min(quality, 50),
DelayState::HighDelay => std::cmp::min(quality, 25),
DelayState::Broken => 10,
};
.map(|(_, u)| u.quality)
.filter(|q| *q != None)
.max_by(|a, b| a.unwrap_or_default().0.cmp(&b.unwrap_or_default().0))
.unwrap_or_default()
.unwrap_or_default()
.1;
let mut quality = latest_quality;
// network delay
if Self::abr_enabled() && typ != Some(RefreshType::SetImageQuality) {
// max delay
let delay = self
.users
.iter()
.map(|u| u.1.delay)
.filter(|d| d.is_some())
.max_by(|a, b| {
(a.unwrap_or_default().state as u32).cmp(&(b.unwrap_or_default().state as u32))
});
let delay = delay.unwrap_or_default().unwrap_or_default().state;
if delay != DelayState::Normal {
match self.quality {
Quality::Best => {
quality = Quality::Balanced;
}
Quality::Balanced => {
quality = Quality::Low;
}
Quality::Low => {
quality = Quality::Low;
}
Quality::Custom(b) => match delay {
DelayState::LowDelay => {
quality =
Quality::Custom(if b >= 150 { 100 } else { std::cmp::min(50, b) });
}
DelayState::HighDelay => {
quality =
Quality::Custom(if b >= 100 { 50 } else { std::cmp::min(25, b) });
}
DelayState::Broken => {
quality =
Quality::Custom(if b >= 50 { 25 } else { std::cmp::min(10, b) });
}
DelayState::Normal => {}
},
}
} else {
match self.quality {
Quality::Low => {
if latest_quality == Quality::Best {
quality = Quality::Balanced;
}
}
Quality::Custom(current_b) => {
if let Quality::Custom(latest_b) = latest_quality {
if current_b < latest_b / 2 {
quality = Quality::Custom(latest_b / 2);
}
}
}
_ => {}
}
}
quality
} else {
ImageQuality::Balanced.as_percent() as _
};
// bitrate
#[allow(unused_mut)]
let mut base_bitrate = ((self.width * self.height) / 800) as u32;
if base_bitrate == 0 {
base_bitrate = 1920 * 1080 / 800;
}
#[cfg(target_os = "android")]
{
// fix when android screen shrinks
let fix = scrap::Display::fix_quality() as u32;
log::debug!("Android screen, fix quality:{}", fix);
base_bitrate = base_bitrate * fix;
}
let target_bitrate = base_bitrate * quality as u32 / 100;
if self.target_bitrate != target_bitrate {
self.target_bitrate = target_bitrate;
updated = true;
}
self.updated = updated;
self.quality = quality;
}
pub fn user_custom_fps(&mut self, id: i32, fps: u32) {
@ -211,7 +241,7 @@ impl VideoQoS {
},
);
}
self.refresh();
self.refresh(None);
}
pub fn user_full_speed_fps(&mut self, id: i32, full_speed_fps: u32) {
@ -226,23 +256,27 @@ impl VideoQoS {
},
);
}
self.refresh();
self.refresh(None);
}
pub fn user_image_quality(&mut self, id: i32, image_quality: i32) {
let convert_quality = |q: i32| -> i32 {
// https://github.com/rustdesk/rustdesk/blob/d716e2b40c38737f1aa3f16de0dec67394a6ac68/src/server/video_service.rs#L493
let convert_quality = |q: i32| {
if q == ImageQuality::Balanced.value() {
100 * 2 / 3
Quality::Balanced
} else if q == ImageQuality::Low.value() {
100 / 2
Quality::Low
} else if q == ImageQuality::Best.value() {
100
Quality::Best
} else {
(q >> 8 & 0xFF) * 2
let mut b = (q >> 8 & 0xFF) * 2;
b = std::cmp::max(b, 10);
b = std::cmp::min(b, 200);
Quality::Custom(b as u32)
}
};
let quality = Some((convert_quality(image_quality), hbb_common::get_time()));
let quality = Some((hbb_common::get_time(), convert_quality(image_quality)));
if let Some(user) = self.users.get_mut(&id) {
user.quality = quality;
} else {
@ -254,43 +288,78 @@ impl VideoQoS {
},
);
}
self.refresh();
self.refresh(Some(RefreshType::SetImageQuality));
}
pub fn user_network_delay(&mut self, id: i32, delay: u32) {
let mut refresh = true;
let state = DelayState::from_delay(delay);
let debounce = 3;
if let Some(user) = self.users.get_mut(&id) {
if let Some((old_state, old_delay, mut counter)) = user.delay {
let new_delay = (delay + old_delay) / 2;
let new_state = DelayState::from_delay(new_delay);
if old_state == new_state {
counter += 1;
if let Some(d) = &mut user.delay {
d.delay = (delay + d.delay) / 2;
let new_state = DelayState::from_delay(d.delay);
let slower_than_old_state = new_state as i32 - d.staging_state as i32;
let slower_than_old_state = if slower_than_old_state > 0 {
Some(true)
} else if slower_than_old_state < 0 {
Some(false)
} else {
counter = 0;
None
};
if d.slower_than_old_state == slower_than_old_state {
let old_counter = d.counter;
d.counter += delay / 1000 + 1;
if old_counter < debounce && d.counter >= debounce {
d.counter = 0;
d.state = d.staging_state;
d.staging_state = new_state;
}
if d.counter % debounce == 0 {
self.refresh(None);
}
} else {
d.counter = 0;
d.staging_state = new_state;
d.slower_than_old_state = slower_than_old_state;
}
let debounce = 3;
refresh = counter == debounce;
user.delay = Some((new_state, new_delay, counter));
} else {
user.delay = Some((state, delay, 0));
user.delay = Some(Delay {
state: DelayState::Normal,
staging_state: state,
delay,
counter: 0,
slower_than_old_state: None,
});
}
} else {
self.users.insert(
id,
UserData {
delay: Some((state, delay, 0)),
delay: Some(Delay {
state: DelayState::Normal,
staging_state: state,
delay,
counter: 0,
slower_than_old_state: None,
}),
..Default::default()
},
);
}
if refresh {
self.refresh();
}
pub fn user_delay_response_elapsed(&mut self, id: i32, elapsed: u128) {
if let Some(user) = self.users.get_mut(&id) {
let old = user.response_delayed;
user.response_delayed = elapsed > 3000;
if old != user.response_delayed {
self.refresh(None);
}
}
}
pub fn on_connection_close(&mut self, id: i32) {
self.users.remove(&id);
self.refresh();
self.refresh(None);
}
}

32
src/server/video_service.rs
View File

@ -514,13 +514,12 @@ fn run(sp: GenericService) -> ResultType<()> {
let mut c = get_capturer(true, last_portable_service_running)?;
let mut video_qos = VIDEO_QOS.lock().unwrap();
video_qos.set_size(c.width as _, c.height as _);
video_qos.refresh();
let mut spf = video_qos.spf();
let bitrate = video_qos.bitrate();
video_qos.refresh(None);
let mut spf;
let mut quality = video_qos.quality();
let abr = VideoQoS::abr_enabled();
drop(video_qos);
log::info!("init bitrate={}, abr enabled:{}", bitrate, abr);
log::info!("init quality={:?}, abr enabled:{}", quality, abr);
let encoder_cfg = match Encoder::negotiated_codec() {
scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => {
@ -528,7 +527,7 @@ fn run(sp: GenericService) -> ResultType<()> {
name,
width: c.width,
height: c.height,
bitrate: bitrate as _,
quality,
})
}
name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => {
@ -536,7 +535,7 @@ fn run(sp: GenericService) -> ResultType<()> {
width: c.width as _,
height: c.height as _,
timebase: [1, 1000], // Output timestamp precision
bitrate,
quality,
codec: if name == scrap::CodecName::VP8 {
VpxVideoCodecId::VP8
} else {
@ -548,7 +547,7 @@ fn run(sp: GenericService) -> ResultType<()> {
scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig {
width: c.width as _,
height: c.height as _,
bitrate: bitrate as _,
quality,
}),
};
@ -558,6 +557,7 @@ fn run(sp: GenericService) -> ResultType<()> {
Err(err) => bail!("Failed to create encoder: {}", err),
}
c.set_use_yuv(encoder.use_yuv());
VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate());
if *SWITCH.lock().unwrap() {
log::debug!("Broadcasting display switch");
@ -611,16 +611,12 @@ fn run(sp: GenericService) -> ResultType<()> {
check_uac_switch(c.privacy_mode_id, c._capturer_privacy_mode_id)?;
let mut video_qos = VIDEO_QOS.lock().unwrap();
if video_qos.check_if_updated() {
log::debug!(
"qos is updated, target_bitrate:{}, fps:{}",
video_qos.bitrate(),
video_qos.fps()
);
if video_qos.bitrate() > 0 {
allow_err!(encoder.set_bitrate(video_qos.bitrate()));
}
spf = video_qos.spf();
spf = video_qos.spf();
if quality != video_qos.quality() {
log::debug!("quality: {:?} -> {:?}", quality, video_qos.quality());
quality = video_qos.quality();
allow_err!(encoder.set_quality(quality));
video_qos.store_bitrate(encoder.bitrate());
}
drop(video_qos);