konevsa/rustdesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat mediacodec: Android H264/H265 decoder support

Browse Source
This commit is contained in:
csf 2022-09-15 20:40:29 +08:00
parent d3bc4a7dc6
commit f310251cfc
7 changed files with 168 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
Cargo.lock generated
View File

@ -3007,6 +3007,20 @@ dependencies = [
"thiserror",
]
[[package]]
name = "ndk"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0"
dependencies = [
"bitflags",
"jni-sys",
"ndk-sys 0.4.0",
"num_enum",
"raw-window-handle 0.5.0",
"thiserror",
]
[[package]]
name = "ndk-context"
version = "0.1.1"
@ -3071,6 +3085,15 @@ dependencies = [
"jni-sys",
]
[[package]]
name = "ndk-sys"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21d83ec9c63ec5bf950200a8e508bdad6659972187b625469f58ef8c08e29046"
dependencies = [
"jni-sys",
]
[[package]]
name = "net2"
version = "0.2.37"
@ -3948,6 +3971,15 @@ dependencies = [
"cty",
]
[[package]]
name = "raw-window-handle"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed7e3d950b66e19e0c372f3fa3fbbcf85b1746b571f74e0c2af6042a5c93420a"
dependencies = [
"cty",
]
[[package]]
name = "rayon"
version = "1.5.3"
@ -4412,6 +4444,7 @@ dependencies = [
"lazy_static",
"libc",
"log",
"ndk 0.7.0",
"num_cpus",
"quest",
"repng",
@ -5857,7 +5890,7 @@ dependencies = [
"objc",
"parking_lot 0.11.2",
"percent-encoding",
"raw-window-handle",
"raw-window-handle 0.4.3",
"smithay-client-toolkit",
"wasm-bindgen",
"wayland-client",

1
Cargo.toml
View File

@ -28,6 +28,7 @@ use_dasp = ["dasp"]
flutter = ["flutter_rust_bridge"]
default = ["use_dasp"]
hwcodec = ["scrap/hwcodec"]
mediacodec = ["scrap/mediacodec"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

2
libs/scrap/Cargo.toml
View File

@ -11,6 +11,7 @@ edition = "2018"
[features]
wayland = ["gstreamer", "gstreamer-app", "gstreamer-video", "dbus", "tracing"]
mediacodec = ["ndk"]
[dependencies]
block = "0.1"
@ -31,6 +32,7 @@ jni = "0.19"
lazy_static = "1.4"
log = "0.4"
serde_json = "1.0"
ndk = { version = "0.7", features = ["media"], optional = true}
[target.'cfg(not(target_os = "android"))'.dev-dependencies]
repng = "0.2"

65
libs/scrap/src/common/codec.rs
View File

@ -7,6 +7,10 @@ use std::{
#[cfg(feature = "hwcodec")]
use crate::hwcodec::*;
#[cfg(feature = "mediacodec")]
use crate::mediacodec::{
MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
};
use crate::vpxcodec::*;
use hbb_common::{
@ -15,7 +19,7 @@ use hbb_common::{
message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState},
ResultType,
};
#[cfg(feature = "hwcodec")]
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
use hbb_common::{
config::{Config2, PeerConfig},
lazy_static,
@ -82,6 +86,8 @@ pub struct Decoder {
hw: HwDecoders,
#[cfg(feature = "hwcodec")]
i420: Vec<u8>,
#[cfg(feature = "mediacodec")]
media_codec: MediaCodecDecoders,
}
#[derive(Debug, Clone)]
@ -242,20 +248,34 @@ impl Decoder {
#[cfg(feature = "hwcodec")]
if check_hwcodec_config() {
let best = HwDecoder::best();
VideoCodecState {
return VideoCodecState {
score_vpx: SCORE_VPX,
score_h264: best.h264.map_or(0, |c| c.score),
score_h265: best.h265.map_or(0, |c| c.score),
perfer: Self::codec_preference(_id).into(),
..Default::default()
}
} else {
};
}
#[cfg(feature = "mediacodec")]
if check_hwcodec_config() {
let score_h264 = if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
92
} else {
0
};
let score_h265 = if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
94
} else {
0
};
return VideoCodecState {
score_vpx: SCORE_VPX,
score_h264,
score_h265,
perfer: Self::codec_preference(_id).into(),
..Default::default()
};
}
#[cfg(not(feature = "hwcodec"))]
VideoCodecState {
score_vpx: SCORE_VPX,
..Default::default()
@ -270,6 +290,8 @@ impl Decoder {
hw: HwDecoder::new_decoders(),
#[cfg(feature = "hwcodec")]
i420: vec![],
#[cfg(feature = "mediacodec")]
media_codec: MediaCodecDecoder::new_decoders(),
}
}
@ -298,6 +320,22 @@ impl Decoder {
Err(anyhow!("don't support h265!"))
}
}
#[cfg(feature = "mediacodec")]
video_frame::Union::H264s(h264s) => {
if let Some(decoder) = &mut self.media_codec.h264 {
Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
} else {
Err(anyhow!("don't support h264!"))
}
}
#[cfg(feature = "mediacodec")]
video_frame::Union::H265s(h265s) => {
if let Some(decoder) = &mut self.media_codec.h265 {
Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
} else {
Err(anyhow!("don't support h265!"))
}
}
_ => Err(anyhow!("unsupported video frame type!")),
}
}
@ -345,7 +383,20 @@ impl Decoder {
return Ok(ret);
}
#[cfg(feature = "hwcodec")]
#[cfg(feature = "mediacodec")]
fn handle_mediacodec_video_frame(
decoder: &mut MediaCodecDecoder,
frames: &EncodedVideoFrames,
rgb: &mut Vec<u8>,
) -> ResultType<bool> {
let mut ret = false;
for h264 in frames.frames.iter() {
return decoder.decode(&h264.data, rgb);
}
return Ok(false);
}
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
fn codec_preference(id: &str) -> PerferCodec {
let codec = PeerConfig::load(id)
.options
@ -363,7 +414,7 @@ impl Decoder {
}
}
#[cfg(feature = "hwcodec")]
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
fn check_hwcodec_config() -> bool {
if let Some(v) = Config2::get().options.get("enable-hwcodec") {
return v != "N";

178
libs/scrap/src/common/mediacodec.rs
View File

@ -1,50 +1,40 @@
use std::{io::Write, time::Duration};
use hbb_common::anyhow::Error;
use hbb_common::{bail, ResultType};
#[cfg(target_os = "android")]
use ndk::media::media_codec::{MediaCodec, MediaCodecDirection, MediaFormat};
use std::ops::Deref;
use std::{
io::Write,
sync::atomic::{AtomicBool, Ordering},
time::Duration,
};
use crate::{
codec::{EncoderApi, EncoderCfg},
I420ToARGB,
};
pub struct MediaCodecEncoder {
encoder: MediaCodec,
}
/// MediaCodec mime type name
const H264_MIME_TYPE: &str = "video/avc";
const H265_MIME_TYPE: &str = "video/hevc";
// const VP8_MIME_TYPE: &str = "video/x-vnd.on2.vp8";
// const VP9_MIME_TYPE: &str = "video/x-vnd.on2.vp9";
impl EncoderApi for MediaCodecEncoder {
fn new(cfg: EncoderCfg) -> ResultType<Self>
where
Self: Sized,
{
if let EncoderCfg::HW(cfg) = cfg {
create_media_codec(&cfg.codec_name, MediaCodecDirection::Encoder)
} else {
bail!("encoder type mismatch")
}
}
// TODO MediaCodecEncoder
fn encode_to_message(
&mut self,
frame: &[u8],
ms: i64,
) -> ResultType<hbb_common::message_proto::Message> {
todo!()
}
fn use_yuv(&self) -> bool {
todo!()
}
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
todo!()
}
}
pub static H264_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false);
pub static H265_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false);
pub struct MediaCodecDecoder {
decoder: MediaCodec,
// pub info: CodecInfo,
name: String,
}
impl Deref for MediaCodecDecoder {
type Target = MediaCodec;
fn deref(&self) -> &Self::Target {
&self.decoder
}
}
pub struct MediaCodecDecoders {
@ -52,85 +42,50 @@ pub struct MediaCodecDecoders {
pub h265: Option<MediaCodecDecoder>,
}
// "video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm)
// "video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm)
// "video/avc" - H.264/AVC video
// "video/hevc" - H.265/HEVC video
impl MediaCodecDecoder {
pub fn new_decoders() -> MediaCodecDecoders {
// 直接生成 h264 和 h265
// 264
let h264 = create_media_codec("video/avc", MediaCodecDirection::Decoder)
.map(|decoder| MediaCodecDecoder { decoder });
let h265 = create_media_codec("video/hevc", MediaCodecDirection::Decoder)
.map(|decoder| MediaCodecDecoder { decoder });
let h264 = create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder);
let h265 = create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder);
MediaCodecDecoders { h264, h265 }
}
pub fn decode(&mut self, data: &[u8], rgb: &mut Vec<u8>) -> ResultType<bool> {
log::debug!("start dequeue_input");
match self
.decoder
.dequeue_input_buffer(Duration::from_millis(10))
.unwrap()
{
match self.dequeue_input_buffer(Duration::from_millis(10))? {
Some(mut input_buffer) => {
let mut buf = input_buffer.buffer_mut();
log::debug!(
"dequeue_input success:buf ptr:{:?},len:{}",
buf.as_ptr(),
buf.len()
);
if data.len() > buf.len() {
log::error!("break! res.len()>buf.len()");
bail!("break! res.len()>buf.len()");
log::error!("Failed to decode, the input data size is bigger than input buf");
bail!("The input data size is bigger than input buf");
}
buf.write_all(&data).unwrap();
if let Err(e) = self
.decoder
.queue_input_buffer(input_buffer, 0, data.len(), 0, 0)
{
log::debug!("debug queue_input_buffer:{:?}", e);
};
buf.write_all(&data)?;
self.queue_input_buffer(input_buffer, 0, data.len(), 0, 0)?;
}
None => {
log::debug!("dequeue_input_buffer fail :None");
log::debug!("Failed to dequeue_input_buffer: No available input_buffer");
}
};
return match self
.decoder
.dequeue_output_buffer(Duration::from_millis(100))
{
Ok(Some(output_buffer)) => {
log::debug!("dequeue_output success");
// let res_format = output_buffer.format();
let res_format = self.decoder.output_format();
log::debug!("res_format:{:?}", res_format.str("mime"));
log::debug!("res_color:{:?}", res_format.i32("color-format"));
log::debug!("stride:{:?}", res_format.i32("stride"));
let w = res_format.i32("width").unwrap() as usize;
let h = res_format.i32("height").unwrap() as usize;
let stride = res_format.i32("stride").unwrap(); // todo
// let w = 1920;
// let h = 1080;
// let stride = 1920; // todo
return match self.dequeue_output_buffer(Duration::from_millis(100))? {
Some(output_buffer) => {
let res_format = self.output_format();
let w = res_format
.i32("width")
.ok_or(Error::msg("Failed to dequeue_output_buffer, width is None"))?
as usize;
let h = res_format.i32("height").ok_or(Error::msg(
"Failed to dequeue_output_buffer, height is None",
))? as usize;
let stride = res_format.i32("stride").ok_or(Error::msg(
"Failed to dequeue_output_buffer, stride is None",
))?;
let buf = output_buffer.buffer();
log::debug!("output_buffer ptr:{:?} len:{}", buf.as_ptr(), buf.len());
let bps = 4;
let u = buf.len() * 2 / 3;
let v = buf.len() * 5 / 6;
rgb.resize(h * w * bps, 0);
log::debug!("start I420ToARGB,u:{},v:{},w:{},h:{}", u, v, w, h);
let y_ptr = buf.as_ptr();
let u_ptr = buf[u..].as_ptr();
let v_ptr = buf[v..].as_ptr();
log::debug!("ptr,y:{:?},u:{:?},v:{:?}", y_ptr, u_ptr, v_ptr);
unsafe {
I420ToARGB(
y_ptr,
@ -145,43 +100,48 @@ impl MediaCodecDecoder {
h as _,
);
}
log::debug!("end I420ToARGB");
log::debug!("release_output_buffer");
self.decoder
.release_output_buffer(output_buffer, false)
.unwrap();
log::debug!("return true");
self.release_output_buffer(output_buffer, false)?;
Ok(true)
}
Ok(None) => {
log::debug!("dequeue_output fail :None");
Ok(false)
}
Err(e) => {
log::debug!("dequeue_output fail :error:{:?}", e);
None => {
log::debug!("Failed to dequeue_output: No available dequeue_output");
Ok(false)
}
};
}
}
fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option<MediaCodec> {
let codec = MediaCodec::from_decoder_type(name).unwrap();
log::debug!("start init");
fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option<MediaCodecDecoder> {
let codec = MediaCodec::from_decoder_type(name)?;
let media_format = MediaFormat::new();
media_format.set_str("mime", name);
media_format.set_i32("width", 0);
media_format.set_i32("height", 0);
media_format.set_i32("color-format", 19); // COLOR_FormatYUV420Planar
if let Err(e) = codec.configure(&media_format, None, direction) {
log::error!("failed to decoder.init:{:?}", e);
log::error!("Failed to init decoder:{:?}", e);
return None;
};
log::error!("decoder init success");
if let Err(e) = codec.start() {
log::error!("failed to decoder.start:{:?}", e);
log::error!("Failed to start decoder:{:?}", e);
return None;
};
log::debug!("init decoder successed!:{:?}", name);
return Some(codec);
log::debug!("Init decoder successed!: {:?}", name);
return Some(MediaCodecDecoder {
decoder: codec,
name: name.to_owned(),
});
}
pub fn check_mediacodec() {
std::thread::spawn(move || {
// check decoders
let decoders = MediaCodecDecoder::new_decoders();
H264_DECODER_SUPPORT.swap(decoders.h264.is_some(), Ordering::SeqCst);
H265_DECODER_SUPPORT.swap(decoders.h265.is_some(), Ordering::SeqCst);
decoders.h264.map(|d| d.stop());
decoders.h265.map(|d| d.stop());
// TODO encoders
});
}

2
libs/scrap/src/common/mod.rs
View File

@ -32,6 +32,8 @@ pub mod codec;
mod convert;
#[cfg(feature = "hwcodec")]
pub mod hwcodec;
#[cfg(feature = "mediacodec")]
pub mod mediacodec;
pub mod vpxcodec;
pub use self::convert::*;
pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller

2
src/flutter_ffi.rs
View File

@ -49,6 +49,8 @@ fn initialize(app_dir: &str) {
.with_min_level(log::Level::Debug) // limit log level
.with_tag("ffi"), // logs will show under mytag tag
);
#[cfg(feature = "mediacodec")]
scrap::mediacodec::check_mediacodec();
}
#[cfg(target_os = "ios")]
{