konevsa/rustdesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix, mac hwcodec decoding align use dst_align (#8215)

Browse Source 
Signed-off-by: 21pages <sunboeasy@gmail.com>
This commit is contained in:
21pages 2024-05-30 23:40:25 +08:00 committed by GitHub
parent d4dda94e2a
commit 8919ea65e3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 114 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
libs/scrap/src/common/convert.rs
View File

@ -13,186 +13,6 @@ use hbb_common::{bail, log, ResultType};
generate_call_macro!(call_yuv, false);
#[cfg(feature = "hwcodec")]
pub mod hw {
use super::*;
use crate::ImageFormat;
#[cfg(target_os = "windows")]
use hwcodec::{ffmpeg::AVPixelFormat, ffmpeg_ram::ffmpeg_linesize_offset_length};
#[cfg(target_os = "windows")]
pub fn hw_nv12_to(
fmt: ImageFormat,
width: usize,
height: usize,
src_y: &[u8],
src_uv: &[u8],
src_stride_y: usize,
src_stride_uv: usize,
dst: &mut Vec<u8>,
i420: &mut Vec<u8>,
align: usize,
) -> ResultType<()> {
let nv12_stride_y = src_stride_y;
let nv12_stride_uv = src_stride_uv;
if let Ok((linesize_i420, offset_i420, i420_len)) =
ffmpeg_linesize_offset_length(AVPixelFormat::AV_PIX_FMT_YUV420P, width, height, align)
{
dst.resize(width * height * 4, 0);
let i420_stride_y = linesize_i420[0];
let i420_stride_u = linesize_i420[1];
let i420_stride_v = linesize_i420[2];
i420.resize(i420_len as _, 0);
let i420_offset_y = unsafe { i420.as_ptr().add(0) as _ };
let i420_offset_u = unsafe { i420.as_ptr().add(offset_i420[0] as _) as _ };
let i420_offset_v = unsafe { i420.as_ptr().add(offset_i420[1] as _) as _ };
call_yuv!(NV12ToI420(
src_y.as_ptr(),
nv12_stride_y as _,
src_uv.as_ptr(),
nv12_stride_uv as _,
i420_offset_y,
i420_stride_y,
i420_offset_u,
i420_stride_u,
i420_offset_v,
i420_stride_v,
width as _,
height as _,
));
match fmt {
ImageFormat::ARGB => {
call_yuv!(I420ToARGB(
i420_offset_y,
i420_stride_y,
i420_offset_u,
i420_stride_u,
i420_offset_v,
i420_stride_v,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
));
}
ImageFormat::ABGR => {
call_yuv!(I420ToABGR(
i420_offset_y,
i420_stride_y,
i420_offset_u,
i420_stride_u,
i420_offset_v,
i420_stride_v,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
));
}
_ => {
bail!("unsupported image format");
}
}
return Ok(());
}
bail!("get linesize offset failed");
}
#[cfg(not(target_os = "windows"))]
pub fn hw_nv12_to(
fmt: ImageFormat,
width: usize,
height: usize,
src_y: &[u8],
src_uv: &[u8],
src_stride_y: usize,
src_stride_uv: usize,
dst: &mut Vec<u8>,
_i420: &mut Vec<u8>,
_align: usize,
) -> ResultType<()> {
dst.resize(width * height * 4, 0);
match fmt {
ImageFormat::ARGB => {
call_yuv!(NV12ToARGB(
src_y.as_ptr(),
src_stride_y as _,
src_uv.as_ptr(),
src_stride_uv as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
));
}
ImageFormat::ABGR => {
call_yuv!(NV12ToABGR(
src_y.as_ptr(),
src_stride_y as _,
src_uv.as_ptr(),
src_stride_uv as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
));
}
_ => bail!("unsupported image format"),
}
Ok(())
}
pub fn hw_i420_to(
fmt: ImageFormat,
width: usize,
height: usize,
src_y: &[u8],
src_u: &[u8],
src_v: &[u8],
src_stride_y: usize,
src_stride_u: usize,
src_stride_v: usize,
dst: &mut Vec<u8>,
) -> ResultType<()> {
let src_y = src_y.as_ptr();
let src_u = src_u.as_ptr();
let src_v = src_v.as_ptr();
dst.resize(width * height * 4, 0);
match fmt {
ImageFormat::ARGB => {
call_yuv!(I420ToARGB(
src_y,
src_stride_y as _,
src_u,
src_stride_u as _,
src_v,
src_stride_v as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
));
}
ImageFormat::ABGR => {
call_yuv!(I420ToABGR(
src_y,
src_stride_y as _,
src_u,
src_stride_u as _,
src_v,
src_stride_v as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
));
}
_ => bail!("unsupported image format"),
};
Ok(())
}
}
#[cfg(not(target_os = "ios"))]
pub fn convert_to_yuv(
captured: &PixelBuffer,

147
libs/scrap/src/common/hwcodec.rs
View File

@ -2,7 +2,8 @@ use crate::{
codec::{
base_bitrate, codec_thread_num, enable_hwcodec_option, EncoderApi, EncoderCfg, Quality as Q,
},
hw, CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
convert::*,
CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
};
use hbb_common::{
anyhow::{anyhow, bail, Context},
@ -23,7 +24,7 @@ use hwcodec::{
ffmpeg_ram::{
decode::{DecodeContext, DecodeFrame, Decoder},
encode::{EncodeContext, EncodeFrame, Encoder},
CodecInfo,
ffmpeg_linesize_offset_length, CodecInfo,
},
};
@ -32,6 +33,8 @@ pub const DEFAULT_TIME_BASE: [i32; 2] = [1, 30];
const DEFAULT_GOP: i32 = i32::MAX;
const DEFAULT_HW_QUALITY: Quality = Quality_Default;
crate::generate_call_macro!(call_yuv, false);
#[derive(Debug, Clone)]
pub struct HwRamEncoderConfig {
pub name: String,
@ -237,9 +240,9 @@ impl HwRamEncoder {
}
}
fn rate_control(config: &HwRamEncoderConfig) -> RateControl {
fn rate_control(_config: &HwRamEncoderConfig) -> RateControl {
#[cfg(target_os = "android")]
if config.name.contains("mediacodec") {
if _config.name.contains("mediacodec") {
return RC_VBR;
}
RC_CBR
@ -262,15 +265,15 @@ impl HwRamEncoder {
quality * factor
}
pub fn check_bitrate_range(config: &HwRamEncoderConfig, bitrate: u32) -> u32 {
pub fn check_bitrate_range(_config: &HwRamEncoderConfig, bitrate: u32) -> u32 {
#[cfg(target_os = "android")]
if config.name.contains("mediacodec") {
if _config.name.contains("mediacodec") {
let info = crate::android::ffi::get_codec_info();
if let Some(info) = info {
if let Some(codec) = info
.codecs
.iter()
.find(|c| Some(c.name.clone()) == config.mc_name && c.is_encoder)
.find(|c| Some(c.name.clone()) == _config.mc_name && c.is_encoder)
{
if codec.max_bitrate > codec.min_bitrate {
if bitrate > codec.max_bitrate {
@ -368,54 +371,100 @@ impl HwRamDecoderImage<'_> {
// rgb [in/out] fmt and stride must be set in ImageRgb
pub fn to_fmt(&self, rgb: &mut ImageRgb, i420: &mut Vec<u8>) -> ResultType<()> {
let frame = self.frame;
rgb.w = frame.width as _;
rgb.h = frame.height as _;
// take dst_stride into account when you convert
let dst_stride = rgb.stride();
let width = frame.width;
let height = frame.height;
rgb.w = width as _;
rgb.h = height as _;
let dst_align = rgb.align();
let bytes_per_row = (rgb.w * 4 + dst_align - 1) & !(dst_align - 1);
rgb.raw.resize(rgb.h * bytes_per_row, 0);
match frame.pixfmt {
AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to(
rgb.fmt(),
frame.width as _,
frame.height as _,
&frame.data[0],
&frame.data[1],
frame.linesize[0] as _,
frame.linesize[1] as _,
&mut rgb.raw as _,
i420,
HW_STRIDE_ALIGN,
)?,
AVPixelFormat::AV_PIX_FMT_NV12 => {
// I420ToARGB is much faster than NV12ToARGB in tests on Windows
if cfg!(windows) {
let Ok((linesize_i420, offset_i420, len_i420)) = ffmpeg_linesize_offset_length(
AVPixelFormat::AV_PIX_FMT_YUV420P,
width as _,
height as _,
HW_STRIDE_ALIGN,
) else {
bail!("failed to get i420 linesize, offset, length");
};
i420.resize(len_i420 as _, 0);
let i420_offset_y = unsafe { i420.as_ptr().add(0) as _ };
let i420_offset_u = unsafe { i420.as_ptr().add(offset_i420[0] as _) as _ };
let i420_offset_v = unsafe { i420.as_ptr().add(offset_i420[1] as _) as _ };
call_yuv!(NV12ToI420(
frame.data[0].as_ptr(),
frame.linesize[0],
frame.data[1].as_ptr(),
frame.linesize[1],
i420_offset_y,
linesize_i420[0],
i420_offset_u,
linesize_i420[1],
i420_offset_v,
linesize_i420[2],
width,
height,
));
let f = match rgb.fmt() {
ImageFormat::ARGB => I420ToARGB,
ImageFormat::ABGR => I420ToABGR,
_ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
};
call_yuv!(f(
i420_offset_y,
linesize_i420[0],
i420_offset_u,
linesize_i420[1],
i420_offset_v,
linesize_i420[2],
rgb.raw.as_mut_ptr(),
bytes_per_row as _,
width,
height,
));
} else {
let f = match rgb.fmt() {
ImageFormat::ARGB => NV12ToARGB,
ImageFormat::ABGR => NV12ToABGR,
_ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
};
call_yuv!(f(
frame.data[0].as_ptr(),
frame.linesize[0],
frame.data[1].as_ptr(),
frame.linesize[1],
rgb.raw.as_mut_ptr(),
bytes_per_row as _,
width,
height,
));
}
}
AVPixelFormat::AV_PIX_FMT_YUV420P => {
hw::hw_i420_to(
rgb.fmt(),
frame.width as _,
frame.height as _,
&frame.data[0],
&frame.data[1],
&frame.data[2],
frame.linesize[0] as _,
frame.linesize[1] as _,
frame.linesize[2] as _,
&mut rgb.raw as _,
)?;
let f = match rgb.fmt() {
ImageFormat::ARGB => I420ToARGB,
ImageFormat::ABGR => I420ToABGR,
_ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
};
call_yuv!(f(
frame.data[0].as_ptr(),
frame.linesize[0],
frame.data[1].as_ptr(),
frame.linesize[1],
frame.data[2].as_ptr(),
frame.linesize[2],
rgb.raw.as_mut_ptr(),
bytes_per_row as _,
width,
height,
));
}
}
Ok(())
}
pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
let mut rgb = ImageRgb::new(ImageFormat::ARGB, 1);
self.to_fmt(&mut rgb, i420)?;
*bgra = rgb.raw;
Ok(())
}
pub fn rgba(&self, rgba: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
let mut rgb = ImageRgb::new(ImageFormat::ABGR, 1);
self.to_fmt(&mut rgb, i420)?;
*rgba = rgb.raw;
Ok(())
}
}
#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]

22
libs/scrap/src/common/mod.rs
View File

@ -53,7 +53,7 @@ pub mod record;
mod vpx;
#[repr(usize)]
#[derive(Copy, Clone)]
#[derive(Debug, Copy, Clone)]
pub enum ImageFormat {
Raw,
ABGR,
@ -65,17 +65,17 @@ pub struct ImageRgb {
pub w: usize,
pub h: usize,
pub fmt: ImageFormat,
pub stride: usize,
pub align: usize,
}
impl ImageRgb {
pub fn new(fmt: ImageFormat, stride: usize) -> Self {
pub fn new(fmt: ImageFormat, align: usize) -> Self {
Self {
raw: Vec::new(),
w: 0,
h: 0,
fmt,
stride,
align,
}
}
@ -85,13 +85,13 @@ impl ImageRgb {
}
#[inline]
pub fn stride(&self) -> usize {
self.stride
pub fn align(&self) -> usize {
self.align
}
#[inline]
pub fn set_stride(&mut self, stride: usize) {
self.stride = stride;
pub fn set_align(&mut self, align: usize) {
self.align = align;
}
}
@ -378,20 +378,20 @@ pub trait GoogleImage {
fn stride(&self) -> Vec<i32>;
fn planes(&self) -> Vec<*mut u8>;
fn chroma(&self) -> Chroma;
fn get_bytes_per_row(w: usize, fmt: ImageFormat, stride: usize) -> usize {
fn get_bytes_per_row(w: usize, fmt: ImageFormat, align: usize) -> usize {
let bytes_per_pixel = match fmt {
ImageFormat::Raw => 3,
ImageFormat::ARGB | ImageFormat::ABGR => 4,
};
// https://github.com/lemenkov/libyuv/blob/6900494d90ae095d44405cd4cc3f346971fa69c9/source/convert_argb.cc#L128
// https://github.com/lemenkov/libyuv/blob/6900494d90ae095d44405cd4cc3f346971fa69c9/source/convert_argb.cc#L129
(w * bytes_per_pixel + stride - 1) & !(stride - 1)
(w * bytes_per_pixel + align - 1) & !(align - 1)
}
// rgb [in/out] fmt and stride must be set in ImageRgb
fn to(&self, rgb: &mut ImageRgb) {
rgb.w = self.width();
rgb.h = self.height();
let bytes_per_row = Self::get_bytes_per_row(rgb.w, rgb.fmt, rgb.stride());
let bytes_per_row = Self::get_bytes_per_row(rgb.w, rgb.fmt, rgb.align());
rgb.raw.resize(rgb.h * bytes_per_row, 0);
let stride = self.stride();
let planes = self.planes();

4
src/client.rs
View File

@ -1052,7 +1052,7 @@ impl VideoHandler {
log::info!("new video handler for display #{_display}, format: {format:?}, luid: {luid:?}");
VideoHandler {
decoder: Decoder::new(format, luid),
rgb: ImageRgb::new(ImageFormat::ARGB, crate::get_dst_stride_rgba()),
rgb: ImageRgb::new(ImageFormat::ARGB, crate::get_dst_align_rgba()),
texture: std::ptr::null_mut(),
recorder: Default::default(),
record: false,
@ -1105,7 +1105,7 @@ impl VideoHandler {
/// Reset the decoder, change format if it is Some
pub fn reset(&mut self, format: Option<CodecFormat>) {
#[cfg(target_os = "macos")]
self.rgb.set_stride(crate::get_dst_stride_rgba());
self.rgb.set_align(crate::get_dst_align_rgba());
let luid = Self::get_adapter_luid();
let format = format.unwrap_or(self.decoder.format());
self.decoder = Decoder::new(format, luid);

4
src/common.rs
View File

@ -1617,7 +1617,7 @@ fn read_custom_client_advanced_settings(
#[inline]
#[cfg(target_os = "macos")]
pub fn get_dst_stride_rgba() -> usize {
pub fn get_dst_align_rgba() -> usize {
// https://developer.apple.com/forums/thread/712709
// Memory alignment should be multiple of 64.
if crate::ui_interface::use_texture_render() {
@ -1629,7 +1629,7 @@ pub fn get_dst_stride_rgba() -> usize {
#[inline]
#[cfg(not(target_os = "macos"))]
pub fn get_dst_stride_rgba() -> usize {
pub fn get_dst_align_rgba() -> usize {
1
}

2
src/flutter.rs
View File

@ -445,7 +445,7 @@ impl VideoRenderer {
rgba.raw.len() as _,
rgba.w as _,
rgba.h as _,
rgba.stride() as _,
rgba.align() as _,
)
};
}

1
src/flutter_ffi.rs
View File

@ -3,7 +3,6 @@ use crate::{
common::{is_keyboard_mode_supported, make_fd_to_json},
flutter::{
self, session_add, session_add_existed, session_start_, sessions, try_sync_peer_option,
FlutterHandler,
},
input::*,
ui_interface::{self, *},