2021-06-22 20:08:08 +00:00
// Copyright (C) 2021 Mathieu Duponchelle <mathieu@centricular.com>
//
2022-01-15 18:40:12 +00:00
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
2021-06-22 20:08:08 +00:00
//
2022-01-15 18:40:12 +00:00
// SPDX-License-Identifier: MPL-2.0
2021-06-22 20:08:08 +00:00
2023-03-01 02:04:44 +00:00
use crate ::cea608utils ::Cea608Mode ;
2023-11-21 13:09:22 +00:00
use crate ::cea708utils ::Cea708Mode ;
2023-03-24 23:14:46 +00:00
use anyhow ::{ anyhow , Error } ;
2021-06-22 20:08:08 +00:00
use gst ::glib ;
use gst ::prelude ::* ;
use gst ::subclass ::prelude ::* ;
2023-03-24 23:14:46 +00:00
use std ::collections ::HashMap ;
2021-06-22 20:08:08 +00:00
use std ::sync ::Mutex ;
2024-01-31 15:07:56 +00:00
use once_cell ::sync ::Lazy ;
2021-06-22 20:08:08 +00:00
2023-12-12 03:20:43 +00:00
use super ::{ CaptionSource , MuxMethod } ;
2022-03-07 16:06:12 +00:00
2021-06-22 20:08:08 +00:00
static CAT : Lazy < gst ::DebugCategory > = Lazy ::new ( | | {
gst ::DebugCategory ::new (
" transcriberbin " ,
gst ::DebugColorFlags ::empty ( ) ,
Some ( " Transcribe and inject closed captions " ) ,
)
} ) ;
const DEFAULT_PASSTHROUGH : bool = false ;
const DEFAULT_LATENCY : gst ::ClockTime = gst ::ClockTime ::from_seconds ( 4 ) ;
2023-03-24 23:14:46 +00:00
const DEFAULT_TRANSLATE_LATENCY : gst ::ClockTime = gst ::ClockTime ::from_mseconds ( 500 ) ;
2021-06-22 20:08:08 +00:00
const DEFAULT_ACCUMULATE : gst ::ClockTime = gst ::ClockTime ::ZERO ;
const DEFAULT_MODE : Cea608Mode = Cea608Mode ::RollUp2 ;
2022-03-07 16:06:12 +00:00
const DEFAULT_CAPTION_SOURCE : CaptionSource = CaptionSource ::Both ;
2023-05-09 14:24:15 +00:00
const DEFAULT_INPUT_LANG_CODE : & str = " en-US " ;
2023-11-21 13:09:22 +00:00
const DEFAULT_MUX_METHOD : MuxMethod = MuxMethod ::Cea608 ;
2021-06-22 20:08:08 +00:00
2023-11-21 13:09:22 +00:00
const CEAX08MUX_LATENCY : gst ::ClockTime = gst ::ClockTime ::from_mseconds ( 100 ) ;
2023-03-24 23:14:46 +00:00
/* One per language, including original */
struct TranscriptionChannel {
2023-04-04 15:38:21 +00:00
bin : gst ::Bin ,
2023-03-24 23:14:46 +00:00
textwrap : gst ::Element ,
2023-11-21 13:09:22 +00:00
tttoceax08 : gst ::Element ,
2023-03-24 23:14:46 +00:00
language : String ,
2023-11-21 13:09:22 +00:00
ccmux_pad_name : String ,
2023-03-24 23:14:46 +00:00
}
impl TranscriptionChannel {
fn link_transcriber ( & self , transcriber : & gst ::Element ) -> Result < ( ) , Error > {
let transcriber_src_pad = match self . language . as_str ( ) {
" transcript " = > transcriber
. static_pad ( " src " )
. ok_or ( anyhow! ( " Failed to retrieve transcription source pad " ) ) ? ,
language = > {
let pad = transcriber
. request_pad_simple ( " translate_src_%u " )
. ok_or ( anyhow! ( " Failed to request translation source pad " ) ) ? ;
pad . set_property ( " language-code " , language ) ;
pad
}
} ;
2023-04-04 15:38:21 +00:00
transcriber_src_pad . link ( & self . bin . static_pad ( " sink " ) . unwrap ( ) ) ? ;
2023-03-24 23:14:46 +00:00
Ok ( ( ) )
}
}
2021-06-22 20:08:08 +00:00
struct State {
2023-11-21 13:09:22 +00:00
mux_method : MuxMethod ,
2021-06-22 20:08:08 +00:00
framerate : Option < gst ::Fraction > ,
tearing_down : bool ,
internal_bin : gst ::Bin ,
audio_queue_passthrough : gst ::Element ,
video_queue : gst ::Element ,
audio_tee : gst ::Element ,
2023-07-24 13:51:07 +00:00
transcriber_resample : gst ::Element ,
2021-06-22 20:08:08 +00:00
transcriber_aconv : gst ::Element ,
transcriber : gst ::Element ,
2023-04-04 16:25:40 +00:00
ccmux : gst ::Element ,
2023-04-07 15:46:45 +00:00
ccmux_filter : gst ::Element ,
2021-06-22 20:08:08 +00:00
cccombiner : gst ::Element ,
transcription_bin : gst ::Bin ,
2023-03-24 23:14:46 +00:00
transcription_channels : HashMap < String , TranscriptionChannel > ,
2021-06-22 20:08:08 +00:00
cccapsfilter : gst ::Element ,
2022-03-07 16:06:12 +00:00
transcription_valve : gst ::Element ,
2021-06-22 20:08:08 +00:00
}
struct Settings {
cc_caps : gst ::Caps ,
latency : gst ::ClockTime ,
2023-03-24 23:14:46 +00:00
translate_latency : gst ::ClockTime ,
2021-06-22 20:08:08 +00:00
passthrough : bool ,
accumulate_time : gst ::ClockTime ,
mode : Cea608Mode ,
2022-03-07 16:06:12 +00:00
caption_source : CaptionSource ,
2023-03-24 23:14:46 +00:00
translation_languages : Option < gst ::Structure > ,
2023-05-09 14:24:15 +00:00
language_code : String ,
2023-11-21 13:09:22 +00:00
mux_method : MuxMethod ,
2021-06-22 20:08:08 +00:00
}
impl Default for Settings {
fn default ( ) -> Self {
Self {
cc_caps : gst ::Caps ::builder ( " closedcaption/x-cea-608 " )
2021-11-06 07:34:10 +00:00
. field ( " format " , " raw " )
2021-06-22 20:08:08 +00:00
. build ( ) ,
passthrough : DEFAULT_PASSTHROUGH ,
latency : DEFAULT_LATENCY ,
2023-03-24 23:14:46 +00:00
translate_latency : DEFAULT_TRANSLATE_LATENCY ,
2021-06-22 20:08:08 +00:00
accumulate_time : DEFAULT_ACCUMULATE ,
mode : DEFAULT_MODE ,
2022-03-07 16:06:12 +00:00
caption_source : DEFAULT_CAPTION_SOURCE ,
2023-03-24 23:14:46 +00:00
translation_languages : None ,
2023-05-09 14:24:15 +00:00
language_code : String ::from ( DEFAULT_INPUT_LANG_CODE ) ,
2023-11-21 13:09:22 +00:00
mux_method : DEFAULT_MUX_METHOD ,
2021-06-22 20:08:08 +00:00
}
}
}
// Struct containing all the element data
pub struct TranscriberBin {
audio_srcpad : gst ::GhostPad ,
video_srcpad : gst ::GhostPad ,
audio_sinkpad : gst ::GhostPad ,
video_sinkpad : gst ::GhostPad ,
state : Mutex < Option < State > > ,
settings : Mutex < Settings > ,
}
impl TranscriberBin {
2023-11-21 13:09:22 +00:00
fn construct_channel_bin (
& self ,
lang : & str ,
mux_method : MuxMethod ,
caption_streams : Vec < String > ,
) -> Result < TranscriptionChannel , Error > {
2023-02-25 17:30:26 +00:00
let bin = gst ::Bin ::new ( ) ;
2023-04-04 15:38:21 +00:00
let queue = gst ::ElementFactory ::make ( " queue " ) . build ( ) ? ;
let textwrap = gst ::ElementFactory ::make ( " textwrap " ) . build ( ) ? ;
2023-11-21 13:09:22 +00:00
let ( tttoceax08 , ccmux_pad_name ) = match mux_method {
MuxMethod ::Cea608 = > {
if caption_streams . len ( ) ! = 1 {
anyhow ::bail! ( " Muxing zero/multiple cea608 streams for the same language is not supported " ) ;
}
(
gst ::ElementFactory ::make ( " tttocea608 " ) . build ( ) ? ,
caption_streams [ 0 ] . clone ( ) ,
)
}
MuxMethod ::Cea708 = > {
if ! ( 1 ..= 2 ) . contains ( & caption_streams . len ( ) ) {
anyhow ::bail! (
" Incorrect number of caption stream names {} for muxing 608/708 " ,
caption_streams . len ( )
) ;
}
let mut service_no = None ;
let mut cea608_channel = None ;
for cc in caption_streams . iter ( ) {
if let Some ( cea608 ) = cc . to_lowercase ( ) . strip_prefix ( " cc " ) {
if cea608_channel . is_some ( ) {
anyhow ::bail! (
" Multiple CEA-608 streams for a language are not supported "
) ;
}
2023-12-12 03:20:43 +00:00
let channel = cea608 . parse ::< u32 > ( ) ? ;
if ( 1 ..= 4 ) . contains ( & channel ) {
cea608_channel = Some ( channel ) ;
} else {
anyhow ::bail! (
" CEA-608 channels only support values between 1 and 4 inclusive "
) ;
}
2023-11-21 13:09:22 +00:00
} else if let Some ( cea708_service ) = cc . strip_prefix ( " 708_ " ) {
if service_no . is_some ( ) {
anyhow ::bail! (
" Multiple CEA-708 streams for a language are not supported "
) ;
}
service_no = Some ( cea708_service . parse ::< u32 > ( ) ? ) ;
} else {
anyhow ::bail! (
" caption service name does not match \' 708_%u \' , or cc1, or cc3 "
) ;
}
}
let service_no = service_no . ok_or ( anyhow! ( " No 708 caption service provided " ) ) ? ;
2023-12-12 03:20:43 +00:00
let mut builder =
gst ::ElementFactory ::make ( " tttocea708 " ) . property ( " service-number " , service_no ) ;
if let Some ( channel ) = cea608_channel {
builder = builder . property ( " cea608-channel " , channel ) ;
}
( builder . build ( ) ? , format! ( " sink_ {} " , service_no ) )
2023-11-21 13:09:22 +00:00
}
} ;
2023-04-04 15:38:21 +00:00
let capsfilter = gst ::ElementFactory ::make ( " capsfilter " ) . build ( ) ? ;
let converter = gst ::ElementFactory ::make ( " ccconverter " ) . build ( ) ? ;
2023-11-21 13:09:22 +00:00
bin . add_many ( [ & queue , & textwrap , & tttoceax08 , & capsfilter , & converter ] ) ? ;
gst ::Element ::link_many ( [ & queue , & textwrap , & tttoceax08 , & capsfilter , & converter ] ) ? ;
2023-04-04 15:38:21 +00:00
queue . set_property ( " max-size-buffers " , 0 u32 ) ;
queue . set_property ( " max-size-time " , 0 u64 ) ;
textwrap . set_property ( " lines " , 2 u32 ) ;
2023-11-21 13:09:22 +00:00
let caps = match mux_method {
MuxMethod ::Cea608 = > gst ::Caps ::builder ( " closedcaption/x-cea-608 " )
2023-04-04 15:38:21 +00:00
. field ( " format " , " raw " )
. field ( " framerate " , gst ::Fraction ::new ( 30000 , 1001 ) )
. build ( ) ,
2023-11-21 13:09:22 +00:00
MuxMethod ::Cea708 = > gst ::Caps ::builder ( " closedcaption/x-cea-708 " )
. field ( " format " , " cc_data " )
. field ( " framerate " , gst ::Fraction ::new ( 30000 , 1001 ) )
. build ( ) ,
} ;
capsfilter . set_property ( " caps " , caps ) ;
2023-04-04 15:38:21 +00:00
2023-05-10 15:02:08 +00:00
let sinkpad = gst ::GhostPad ::with_target ( & queue . static_pad ( " sink " ) . unwrap ( ) ) . unwrap ( ) ;
let srcpad = gst ::GhostPad ::with_target ( & converter . static_pad ( " src " ) . unwrap ( ) ) . unwrap ( ) ;
2023-04-04 15:38:21 +00:00
bin . add_pad ( & sinkpad ) ? ;
bin . add_pad ( & srcpad ) ? ;
Ok ( TranscriptionChannel {
bin ,
textwrap ,
2023-11-21 13:09:22 +00:00
tttoceax08 ,
2023-04-04 15:38:21 +00:00
language : String ::from ( lang ) ,
2023-11-21 13:09:22 +00:00
ccmux_pad_name ,
2023-04-04 15:38:21 +00:00
} )
}
2022-10-09 13:06:59 +00:00
fn construct_transcription_bin ( & self , state : & mut State ) -> Result < ( ) , Error > {
gst ::debug! ( CAT , imp : self , " Building transcription bin " ) ;
2021-06-22 20:08:08 +00:00
2022-10-19 16:18:43 +00:00
let aqueue_transcription = gst ::ElementFactory ::make ( " queue " )
. name ( " transqueue " )
. property ( " max-size-buffers " , 0 u32 )
. property ( " max-size-bytes " , 0 u32 )
. property ( " max-size-time " , 5_000_000_000 u64 )
. property_from_str ( " leaky " , " downstream " )
. build ( ) ? ;
let ccconverter = gst ::ElementFactory ::make ( " ccconverter " ) . build ( ) ? ;
2021-06-22 20:08:08 +00:00
2023-03-09 15:30:57 +00:00
state . transcription_bin . add_many ( [
2021-06-22 20:08:08 +00:00
& aqueue_transcription ,
2023-07-24 13:51:07 +00:00
& state . transcriber_resample ,
2021-06-22 20:08:08 +00:00
& state . transcriber_aconv ,
& state . transcriber ,
2023-04-04 16:25:40 +00:00
& state . ccmux ,
2023-04-07 15:46:45 +00:00
& state . ccmux_filter ,
2021-06-22 20:08:08 +00:00
& ccconverter ,
& state . cccapsfilter ,
2022-03-07 16:06:12 +00:00
& state . transcription_valve ,
2021-06-22 20:08:08 +00:00
] ) ? ;
2023-03-09 15:30:57 +00:00
gst ::Element ::link_many ( [
2021-06-22 20:08:08 +00:00
& aqueue_transcription ,
2023-07-24 13:51:07 +00:00
& state . transcriber_resample ,
2021-06-22 20:08:08 +00:00
& state . transcriber_aconv ,
& state . transcriber ,
2023-03-24 23:14:46 +00:00
] ) ? ;
gst ::Element ::link_many ( [
2023-04-04 16:25:40 +00:00
& state . ccmux ,
2023-04-07 15:46:45 +00:00
& state . ccmux_filter ,
2021-06-22 20:08:08 +00:00
& ccconverter ,
& state . cccapsfilter ,
2022-03-07 16:06:12 +00:00
& state . transcription_valve ,
2021-06-22 20:08:08 +00:00
] ) ? ;
2023-11-21 13:09:22 +00:00
for channel in state . transcription_channels . values ( ) {
2023-04-04 15:38:21 +00:00
state . transcription_bin . add ( & channel . bin ) ? ;
2023-03-24 23:14:46 +00:00
channel . link_transcriber ( & state . transcriber ) ? ;
2023-04-04 16:25:40 +00:00
let ccmux_pad = state
. ccmux
2023-11-21 13:09:22 +00:00
. request_pad_simple ( & channel . ccmux_pad_name )
2023-03-24 23:14:46 +00:00
. ok_or ( anyhow! ( " Failed to request ccmux sink pad " ) ) ? ;
2023-04-04 15:38:21 +00:00
channel . bin . static_pad ( " src " ) . unwrap ( ) . link ( & ccmux_pad ) ? ;
2023-03-24 23:14:46 +00:00
}
2023-11-21 13:09:22 +00:00
state . ccmux . set_property ( " latency " , CEAX08MUX_LATENCY ) ;
2023-03-24 23:14:46 +00:00
2023-05-10 15:02:08 +00:00
let transcription_audio_sinkpad =
gst ::GhostPad ::with_target ( & aqueue_transcription . static_pad ( " sink " ) . unwrap ( ) ) . unwrap ( ) ;
let transcription_audio_srcpad =
gst ::GhostPad ::with_target ( & state . transcription_valve . static_pad ( " src " ) . unwrap ( ) )
. unwrap ( ) ;
2021-06-22 20:08:08 +00:00
state
. transcription_bin
. add_pad ( & transcription_audio_sinkpad ) ? ;
state
. transcription_bin
. add_pad ( & transcription_audio_srcpad ) ? ;
state . internal_bin . add ( & state . transcription_bin ) ? ;
state . transcription_bin . set_locked_state ( true ) ;
Ok ( ( ) )
}
2022-10-09 13:06:59 +00:00
fn construct_internal_bin ( & self , state : & mut State ) -> Result < ( ) , Error > {
2022-10-19 16:18:43 +00:00
let aclocksync = gst ::ElementFactory ::make ( " clocksync " ) . build ( ) ? ;
2021-06-22 20:08:08 +00:00
2022-10-19 16:18:43 +00:00
let vclocksync = gst ::ElementFactory ::make ( " clocksync " ) . build ( ) ? ;
2021-06-22 20:08:08 +00:00
2023-03-09 15:30:57 +00:00
state . internal_bin . add_many ( [
2021-06-22 20:08:08 +00:00
& aclocksync ,
& state . audio_tee ,
& state . audio_queue_passthrough ,
& vclocksync ,
& state . video_queue ,
& state . cccombiner ,
] ) ? ;
aclocksync . link ( & state . audio_tee ) ? ;
state
. audio_tee
. link_pads ( Some ( " src_%u " ) , & state . audio_queue_passthrough , Some ( " sink " ) ) ? ;
vclocksync . link ( & state . video_queue ) ? ;
state
. video_queue
. link_pads ( Some ( " src " ) , & state . cccombiner , Some ( " sink " ) ) ? ;
2023-05-10 15:02:08 +00:00
let internal_audio_sinkpad =
gst ::GhostPad ::builder_with_target ( & aclocksync . static_pad ( " sink " ) . unwrap ( ) )
. unwrap ( )
. name ( " audio_sink " )
. build ( ) ;
let internal_audio_srcpad = gst ::GhostPad ::builder_with_target (
2021-06-22 20:08:08 +00:00
& state . audio_queue_passthrough . static_pad ( " src " ) . unwrap ( ) ,
2023-05-10 15:02:08 +00:00
)
. unwrap ( )
. name ( " audio_src " )
. build ( ) ;
let internal_video_sinkpad =
gst ::GhostPad ::builder_with_target ( & vclocksync . static_pad ( " sink " ) . unwrap ( ) )
. unwrap ( )
. name ( " video_sink " )
. build ( ) ;
let internal_video_srcpad =
gst ::GhostPad ::builder_with_target ( & state . cccombiner . static_pad ( " src " ) . unwrap ( ) )
. unwrap ( )
. name ( " video_src " )
. build ( ) ;
2021-06-22 20:08:08 +00:00
state . internal_bin . add_pad ( & internal_audio_sinkpad ) ? ;
state . internal_bin . add_pad ( & internal_audio_srcpad ) ? ;
state . internal_bin . add_pad ( & internal_video_sinkpad ) ? ;
state . internal_bin . add_pad ( & internal_video_srcpad ) ? ;
2022-10-09 13:06:59 +00:00
let imp_weak = self . downgrade ( ) ;
2022-03-07 16:06:12 +00:00
let comp_sinkpad = & state . cccombiner . static_pad ( " sink " ) . unwrap ( ) ;
// Drop caption meta from video buffer if user preference is transcription
comp_sinkpad . add_probe ( gst ::PadProbeType ::BUFFER , move | _ , probe_info | {
2023-10-30 09:34:35 +00:00
let Some ( imp ) = imp_weak . upgrade ( ) else {
return gst ::PadProbeReturn ::Remove ;
2022-03-07 16:06:12 +00:00
} ;
2022-10-09 13:06:59 +00:00
let settings = imp . settings . lock ( ) . unwrap ( ) ;
2022-03-07 16:06:12 +00:00
if settings . caption_source ! = CaptionSource ::Transcription {
return gst ::PadProbeReturn ::Pass ;
}
2023-10-16 16:16:52 +00:00
if let Some ( buffer ) = probe_info . buffer_mut ( ) {
2022-03-07 16:06:12 +00:00
let buffer = buffer . make_mut ( ) ;
while let Some ( meta ) = buffer . meta_mut ::< gst_video ::VideoCaptionMeta > ( ) {
meta . remove ( ) . unwrap ( ) ;
}
}
gst ::PadProbeReturn ::Ok
} ) ;
2022-10-23 20:03:22 +00:00
self . obj ( ) . add ( & state . internal_bin ) ? ;
2021-06-22 20:08:08 +00:00
2022-10-17 17:48:43 +00:00
state . cccombiner . set_property ( " latency " , 100. mseconds ( ) ) ;
2021-06-22 20:08:08 +00:00
self . audio_sinkpad
. set_target ( Some ( & state . internal_bin . static_pad ( " audio_sink " ) . unwrap ( ) ) ) ? ;
self . audio_srcpad
. set_target ( Some ( & state . internal_bin . static_pad ( " audio_src " ) . unwrap ( ) ) ) ? ;
self . video_sinkpad
. set_target ( Some ( & state . internal_bin . static_pad ( " video_sink " ) . unwrap ( ) ) ) ? ;
self . video_srcpad
. set_target ( Some ( & state . internal_bin . static_pad ( " video_src " ) . unwrap ( ) ) ) ? ;
2022-10-09 13:06:59 +00:00
self . construct_transcription_bin ( state ) ? ;
2021-06-22 20:08:08 +00:00
Ok ( ( ) )
}
2022-10-09 13:06:59 +00:00
fn setup_transcription ( & self , state : & State ) {
2021-06-22 20:08:08 +00:00
let settings = self . settings . lock ( ) . unwrap ( ) ;
let mut cc_caps = settings . cc_caps . clone ( ) ;
let cc_caps_mut = cc_caps . make_mut ( ) ;
let s = cc_caps_mut . structure_mut ( 0 ) . unwrap ( ) ;
2022-11-01 08:27:48 +00:00
s . set ( " framerate " , state . framerate . unwrap ( ) ) ;
2021-06-22 20:08:08 +00:00
2021-11-08 09:55:40 +00:00
state . cccapsfilter . set_property ( " caps " , & cc_caps ) ;
2021-06-22 20:08:08 +00:00
2023-11-21 13:09:22 +00:00
let ccmux_caps = match state . mux_method {
MuxMethod ::Cea608 = > gst ::Caps ::builder ( " closedcaption/x-cea-608 " )
. field ( " framerate " , state . framerate . unwrap ( ) )
. build ( ) ,
MuxMethod ::Cea708 = > gst ::Caps ::builder ( " closedcaption/x-cea-708 " )
. field ( " format " , " cc_data " )
. field ( " framerate " , state . framerate . unwrap ( ) )
. build ( ) ,
} ;
2023-04-07 15:46:45 +00:00
state . ccmux_filter . set_property ( " caps " , ccmux_caps ) ;
2023-03-24 23:14:46 +00:00
let max_size_time = settings . latency
+ settings . translate_latency
+ settings . accumulate_time
2023-11-21 13:09:22 +00:00
+ CEAX08MUX_LATENCY ;
2021-06-22 20:08:08 +00:00
2022-11-01 08:27:48 +00:00
for queue in [ & state . audio_queue_passthrough , & state . video_queue ] {
2021-11-08 09:55:40 +00:00
queue . set_property ( " max-size-bytes " , 0 u32 ) ;
queue . set_property ( " max-size-buffers " , 0 u32 ) ;
queue . set_property ( " max-size-time " , max_size_time ) ;
2021-06-22 20:08:08 +00:00
}
let latency_ms = settings . latency . mseconds ( ) as u32 ;
2021-11-08 09:55:40 +00:00
state . transcriber . set_property ( " latency " , latency_ms ) ;
2021-06-22 20:08:08 +00:00
2023-03-24 23:14:46 +00:00
let translate_latency_ms = settings . translate_latency . mseconds ( ) as u32 ;
state
. transcriber
. set_property ( " translate-latency " , translate_latency_ms ) ;
2021-06-22 20:08:08 +00:00
if ! settings . passthrough {
state
. transcription_bin
. link_pads ( Some ( " src " ) , & state . cccombiner , Some ( " caption " ) )
. unwrap ( ) ;
state . transcription_bin . set_locked_state ( false ) ;
state . transcription_bin . sync_state_with_parent ( ) . unwrap ( ) ;
2023-03-22 22:39:32 +00:00
let transcription_sink_pad = state . transcription_bin . static_pad ( " sink " ) . unwrap ( ) ;
2023-04-04 16:25:40 +00:00
// Might be linked already if "translation-languages" is set
if transcription_sink_pad . peer ( ) . is_none ( ) {
let audio_tee_pad = state . audio_tee . request_pad_simple ( " src_%u " ) . unwrap ( ) ;
audio_tee_pad . link ( & transcription_sink_pad ) . unwrap ( ) ;
}
2021-06-22 20:08:08 +00:00
}
drop ( settings ) ;
2022-10-09 13:06:59 +00:00
self . setup_cc_mode ( state ) ;
2021-06-22 20:08:08 +00:00
}
2022-10-09 13:06:59 +00:00
fn disable_transcription_bin ( & self ) {
2021-06-22 20:08:08 +00:00
let mut state = self . state . lock ( ) . unwrap ( ) ;
if let Some ( ref mut state ) = state . as_mut ( ) {
state . tearing_down = false ;
// At this point, we want to check whether passthrough
// has been unset in the meantime
let passthrough = self . settings . lock ( ) . unwrap ( ) . passthrough ;
if passthrough {
2022-10-09 13:06:59 +00:00
gst ::debug! ( CAT , imp : self , " disabling transcription bin " ) ;
2021-07-15 20:38:07 +00:00
2021-06-22 20:08:08 +00:00
let bin_sink_pad = state . transcription_bin . static_pad ( " sink " ) . unwrap ( ) ;
if let Some ( audio_tee_pad ) = bin_sink_pad . peer ( ) {
audio_tee_pad . unlink ( & bin_sink_pad ) . unwrap ( ) ;
state . audio_tee . release_request_pad ( & audio_tee_pad ) ;
}
let bin_src_pad = state . transcription_bin . static_pad ( " src " ) . unwrap ( ) ;
if let Some ( cccombiner_pad ) = bin_src_pad . peer ( ) {
bin_src_pad . unlink ( & cccombiner_pad ) . unwrap ( ) ;
state . cccombiner . release_request_pad ( & cccombiner_pad ) ;
}
state . transcription_bin . set_locked_state ( true ) ;
state . transcription_bin . set_state ( gst ::State ::Null ) . unwrap ( ) ;
}
}
}
2022-10-09 13:06:59 +00:00
fn block_and_update ( & self , passthrough : bool ) {
2021-06-22 20:08:08 +00:00
let mut s = self . state . lock ( ) . unwrap ( ) ;
if let Some ( ref mut state ) = s . as_mut ( ) {
if passthrough {
let sinkpad = state . transcription_bin . static_pad ( " sink " ) . unwrap ( ) ;
2022-10-09 13:06:59 +00:00
let imp_weak = self . downgrade ( ) ;
2021-06-22 20:08:08 +00:00
state . tearing_down = true ;
drop ( s ) ;
let _ = sinkpad . add_probe (
gst ::PadProbeType ::IDLE
| gst ::PadProbeType ::BUFFER
| gst ::PadProbeType ::EVENT_DOWNSTREAM ,
move | _pad , _info | {
2023-10-30 09:34:35 +00:00
let Some ( imp ) = imp_weak . upgrade ( ) else {
return gst ::PadProbeReturn ::Remove ;
2021-06-22 20:08:08 +00:00
} ;
2022-10-09 13:06:59 +00:00
imp . disable_transcription_bin ( ) ;
2021-06-22 20:08:08 +00:00
gst ::PadProbeReturn ::Remove
} ,
) ;
} else if state . tearing_down {
// Do nothing, wait for the previous transcription bin
// to finish tearing down
} else {
state
. transcription_bin
. link_pads ( Some ( " src " ) , & state . cccombiner , Some ( " caption " ) )
. unwrap ( ) ;
state . transcription_bin . set_locked_state ( false ) ;
state . transcription_bin . sync_state_with_parent ( ) . unwrap ( ) ;
2022-04-07 12:05:30 +00:00
let audio_tee_pad = state . audio_tee . request_pad_simple ( " src_%u " ) . unwrap ( ) ;
let transcription_sink_pad = state . transcription_bin . static_pad ( " sink " ) . unwrap ( ) ;
audio_tee_pad . link ( & transcription_sink_pad ) . unwrap ( ) ;
2021-06-22 20:08:08 +00:00
}
}
}
2022-10-09 13:06:59 +00:00
fn setup_cc_mode ( & self , state : & State ) {
2021-06-22 20:08:08 +00:00
let mode = self . settings . lock ( ) . unwrap ( ) . mode ;
2022-10-09 13:06:59 +00:00
gst ::debug! ( CAT , imp : self , " setting CC mode {:?} " , mode ) ;
2021-06-22 20:08:08 +00:00
2023-03-24 23:14:46 +00:00
for channel in state . transcription_channels . values ( ) {
2023-11-21 13:09:22 +00:00
match state . mux_method {
MuxMethod ::Cea608 = > channel . tttoceax08 . set_property ( " mode " , mode ) ,
MuxMethod ::Cea708 = > match mode {
Cea608Mode ::PopOn = > channel . tttoceax08 . set_property ( " mode " , Cea708Mode ::PopOn ) ,
Cea608Mode ::PaintOn = > {
channel . tttoceax08 . set_property ( " mode " , Cea708Mode ::PaintOn )
}
Cea608Mode ::RollUp2 | Cea608Mode ::RollUp3 | Cea608Mode ::RollUp4 = > {
channel . tttoceax08 . set_property ( " mode " , Cea708Mode ::RollUp )
}
} ,
}
2021-06-22 20:08:08 +00:00
2023-03-24 23:14:46 +00:00
if mode . is_rollup ( ) {
channel . textwrap . set_property ( " accumulate-time " , 0 u64 ) ;
} else {
let accumulate_time = self . settings . lock ( ) . unwrap ( ) . accumulate_time ;
2021-06-22 20:08:08 +00:00
2023-03-24 23:14:46 +00:00
channel
. textwrap
. set_property ( " accumulate-time " , accumulate_time ) ;
}
2021-06-22 20:08:08 +00:00
}
}
/* We make no ceremonies here because the function can only
* be called in READY * /
fn relink_transcriber (
& self ,
state : & mut State ,
old_transcriber : & gst ::Element ,
) -> Result < ( ) , Error > {
2023-03-24 23:14:46 +00:00
gst ::debug! (
2021-06-22 20:08:08 +00:00
CAT ,
2022-10-09 13:06:59 +00:00
imp : self ,
2021-06-22 20:08:08 +00:00
" Relinking transcriber, old: {:?}, new: {:?} " ,
old_transcriber ,
state . transcriber
) ;
state . transcriber_aconv . unlink ( old_transcriber ) ;
2023-03-24 23:14:46 +00:00
for channel in state . transcription_channels . values ( ) {
2023-04-04 15:38:21 +00:00
old_transcriber . unlink ( & channel . bin ) ;
2023-03-24 23:14:46 +00:00
}
2021-06-22 20:08:08 +00:00
state . transcription_bin . remove ( old_transcriber ) . unwrap ( ) ;
old_transcriber . set_state ( gst ::State ::Null ) . unwrap ( ) ;
state . transcription_bin . add ( & state . transcriber ) ? ;
state . transcriber . sync_state_with_parent ( ) . unwrap ( ) ;
2023-03-24 23:14:46 +00:00
state . transcriber_aconv . link ( & state . transcriber ) ? ;
for channel in state . transcription_channels . values ( ) {
channel . link_transcriber ( & state . transcriber ) ? ;
}
2021-06-22 20:08:08 +00:00
Ok ( ( ) )
}
2023-11-21 13:09:22 +00:00
fn construct_transcription_channels (
& self ,
settings : & Settings ,
mux_method : MuxMethod ,
transcription_channels : & mut HashMap < String , TranscriptionChannel > ,
) -> Result < ( ) , Error > {
if let Some ( ref map ) = settings . translation_languages {
for ( key , value ) in map . iter ( ) {
let key = key . to_lowercase ( ) ;
let ( language_code , caption_streams ) = match mux_method {
MuxMethod ::Cea608 = > {
if [ " cc1 " , " cc3 " ] . contains ( & key . as_str ( ) ) {
( value . get ::< String > ( ) ? , vec! [ key . to_string ( ) ] )
} else if let Ok ( caption_stream ) = value . get ::< String > ( ) {
if ! [ " cc1 " , " cc3 " ] . contains ( & caption_stream . as_str ( ) ) {
anyhow ::bail! (
" Unknown 608 channel {}, valid values are cc1, cc3 " ,
caption_stream
) ;
}
( key , vec! [ caption_stream ] )
} else {
anyhow ::bail! ( " Unknown 608 channel/language {} " , key ) ;
}
}
MuxMethod ::Cea708 = > {
if let Ok ( caption_stream ) = value . get ::< String > ( ) {
( key , vec! [ caption_stream ] )
} else if let Ok ( caption_streams ) = value . get ::< gst ::List > ( ) {
let mut streams = vec! [ ] ;
for s in caption_streams . iter ( ) {
let service = s . get ::< String > ( ) ? ;
if [ " cc1 " , " cc3 " ] . contains ( & service . as_str ( ) )
| | service . starts_with ( " 708_ " )
{
streams . push ( service ) ;
} else {
anyhow ::bail! ( " Unknown 708 service {}, valid values are cc1, cc3 or 708_* " , key ) ;
}
}
( key , streams )
} else {
anyhow ::bail! ( " Unknown 708 translation language field {} " , key ) ;
}
}
} ;
transcription_channels . insert (
language_code . to_owned ( ) ,
self . construct_channel_bin ( & language_code , mux_method , caption_streams )
. unwrap ( ) ,
) ;
}
} else {
let caption_streams = match mux_method {
MuxMethod ::Cea608 = > vec! [ " cc1 " . to_string ( ) ] ,
MuxMethod ::Cea708 = > vec! [ " cc1 " . to_string ( ) , " 708_1 " . to_string ( ) ] ,
} ;
transcription_channels . insert (
" transcript " . to_string ( ) ,
self . construct_channel_bin ( " transcript " , mux_method , caption_streams )
. unwrap ( ) ,
) ;
}
Ok ( ( ) )
}
2023-05-09 14:24:15 +00:00
fn reconfigure_transcription_bin ( & self , lang_code_only : bool ) -> Result < ( ) , Error > {
2023-04-04 16:25:40 +00:00
let mut state = self . state . lock ( ) . unwrap ( ) ;
if let Some ( ref mut state ) = state . as_mut ( ) {
let settings = self . settings . lock ( ) . unwrap ( ) ;
gst ::debug! (
CAT ,
imp : self ,
2023-05-09 14:24:15 +00:00
" Updating transcription/translation language "
2023-04-04 16:25:40 +00:00
) ;
// Unlink sinkpad temporarily
let sinkpad = state . transcription_bin . static_pad ( " sink " ) . unwrap ( ) ;
let peer = sinkpad . peer ( ) ;
if let Some ( peer ) = & peer {
gst ::debug! ( CAT , imp : self , " Unlinking {:?} " , peer ) ;
peer . unlink ( & sinkpad ) ? ;
state . audio_tee . release_request_pad ( peer ) ;
}
state . transcription_bin . set_locked_state ( true ) ;
state . transcription_bin . set_state ( gst ::State ::Null ) . unwrap ( ) ;
2023-05-09 14:24:15 +00:00
state
. transcriber
. set_property ( " language-code " , & settings . language_code ) ;
if lang_code_only {
if ! settings . passthrough {
gst ::debug! ( CAT , imp : self , " Syncing state with parent " ) ;
drop ( settings ) ;
state . transcription_bin . set_locked_state ( false ) ;
state . transcription_bin . sync_state_with_parent ( ) ? ;
let audio_tee_pad = state . audio_tee . request_pad_simple ( " src_%u " ) . unwrap ( ) ;
audio_tee_pad . link ( & sinkpad ) ? ;
}
return Ok ( ( ) ) ;
}
2023-04-04 16:25:40 +00:00
for channel in state . transcription_channels . values ( ) {
let sinkpad = channel . bin . static_pad ( " sink " ) . unwrap ( ) ;
if let Some ( peer ) = sinkpad . peer ( ) {
peer . unlink ( & sinkpad ) ? ;
if channel . language ! = " transcript " {
state . transcriber . release_request_pad ( & peer ) ;
}
}
let srcpad = channel . bin . static_pad ( " src " ) . unwrap ( ) ;
if let Some ( peer ) = srcpad . peer ( ) {
srcpad . unlink ( & peer ) ? ;
state . ccmux . release_request_pad ( & peer ) ;
}
state . transcription_bin . remove ( & channel . bin ) ? ;
}
state . transcription_channels . clear ( ) ;
2023-11-21 13:09:22 +00:00
self . construct_transcription_channels (
& settings ,
state . mux_method ,
& mut state . transcription_channels ,
) ? ;
2023-04-04 16:25:40 +00:00
2023-11-21 13:09:22 +00:00
for channel in state . transcription_channels . values ( ) {
2023-04-04 16:25:40 +00:00
state . transcription_bin . add ( & channel . bin ) ? ;
channel . link_transcriber ( & state . transcriber ) ? ;
let ccmux_pad = state
. ccmux
2023-11-21 13:09:22 +00:00
. request_pad_simple ( & channel . ccmux_pad_name )
2023-04-04 16:25:40 +00:00
. ok_or ( anyhow! ( " Failed to request ccmux sink pad " ) ) ? ;
channel . bin . static_pad ( " src " ) . unwrap ( ) . link ( & ccmux_pad ) ? ;
}
drop ( settings ) ;
self . setup_cc_mode ( state ) ;
if ! self . settings . lock ( ) . unwrap ( ) . passthrough {
gst ::debug! ( CAT , imp : self , " Syncing state with parent " ) ;
state . transcription_bin . set_locked_state ( false ) ;
state . transcription_bin . sync_state_with_parent ( ) ? ;
let audio_tee_pad = state . audio_tee . request_pad_simple ( " src_%u " ) . unwrap ( ) ;
audio_tee_pad . link ( & sinkpad ) ? ;
}
}
Ok ( ( ) )
}
2023-05-09 14:24:15 +00:00
fn update_languages ( & self , lang_code_only : bool ) {
2023-04-04 16:25:40 +00:00
let s = self . state . lock ( ) . unwrap ( ) ;
if let Some ( state ) = s . as_ref ( ) {
gst ::debug! (
CAT ,
imp : self ,
2023-05-09 14:24:15 +00:00
" Schedule transcription/translation language update "
2023-04-04 16:25:40 +00:00
) ;
let sinkpad = state . transcription_bin . static_pad ( " sink " ) . unwrap ( ) ;
let imp_weak = self . downgrade ( ) ;
drop ( s ) ;
let _ = sinkpad . add_probe (
gst ::PadProbeType ::IDLE
| gst ::PadProbeType ::BUFFER
| gst ::PadProbeType ::EVENT_DOWNSTREAM ,
move | _pad , _info | {
2023-10-30 09:34:35 +00:00
let Some ( imp ) = imp_weak . upgrade ( ) else {
return gst ::PadProbeReturn ::Remove ;
2023-04-04 16:25:40 +00:00
} ;
2023-05-09 14:24:15 +00:00
if imp . reconfigure_transcription_bin ( lang_code_only ) . is_err ( ) {
2023-04-04 16:25:40 +00:00
gst ::element_imp_error! (
imp ,
gst ::StreamError ::Failed ,
[ " Couldn't reconfigure channels " ]
) ;
}
gst ::PadProbeReturn ::Remove
} ,
) ;
} else {
gst ::debug! ( CAT , imp : self , " Transcriber is not configured yet " ) ;
}
}
2021-09-08 12:33:31 +00:00
#[ allow(clippy::single_match) ]
2022-10-09 13:06:59 +00:00
fn src_query ( & self , pad : & gst ::Pad , query : & mut gst ::QueryRef ) -> bool {
2022-01-19 13:07:45 +00:00
use gst ::QueryViewMut ;
2021-06-22 20:08:08 +00:00
2022-02-21 17:43:46 +00:00
gst ::log! ( CAT , obj : pad , " Handling query {:?} " , query ) ;
2021-06-22 20:08:08 +00:00
match query . view_mut ( ) {
2022-01-19 13:07:45 +00:00
QueryViewMut ::Latency ( q ) = > {
2021-06-22 20:08:08 +00:00
let mut upstream_query = gst ::query ::Latency ::new ( ) ;
2022-10-23 20:03:22 +00:00
let ret = gst ::Pad ::query_default ( pad , Some ( & * self . obj ( ) ) , & mut upstream_query ) ;
2021-06-22 20:08:08 +00:00
if ret {
let ( _ , mut min , _ ) = upstream_query . result ( ) ;
2023-03-24 23:14:46 +00:00
let ( received_framerate , translating ) = {
2021-06-22 20:08:08 +00:00
let state = self . state . lock ( ) . unwrap ( ) ;
if let Some ( state ) = state . as_ref ( ) {
2023-03-24 23:14:46 +00:00
(
state . framerate ,
state
. transcription_channels
. values ( )
. any ( | c | c . language ! = " transcript " ) ,
)
2021-06-22 20:08:08 +00:00
} else {
2023-03-24 23:14:46 +00:00
( None , false )
2021-06-22 20:08:08 +00:00
}
} ;
2022-03-08 13:38:55 +00:00
let settings = self . settings . lock ( ) . unwrap ( ) ;
2023-03-24 23:14:46 +00:00
if settings . passthrough | | received_framerate . is_none ( ) {
2023-11-21 13:09:22 +00:00
min + = settings . latency + settings . accumulate_time + CEAX08MUX_LATENCY ;
2023-03-24 23:14:46 +00:00
if translating {
min + = settings . translate_latency ;
}
2023-11-21 13:09:22 +00:00
/* The sub latency introduced by ceax08mux */
2023-03-24 23:14:46 +00:00
if let Some ( framerate ) = received_framerate {
min + = gst ::ClockTime ::SECOND
. mul_div_floor ( framerate . denom ( ) as u64 , framerate . numer ( ) as u64 )
. unwrap ( ) ;
}
2021-06-22 20:08:08 +00:00
} else if settings . mode . is_rollup ( ) {
min + = settings . accumulate_time ;
}
q . set ( true , min , gst ::ClockTime ::NONE ) ;
}
ret
}
2022-10-23 20:03:22 +00:00
_ = > gst ::Pad ::query_default ( pad , Some ( & * self . obj ( ) ) , query ) ,
2021-06-22 20:08:08 +00:00
}
}
fn build_state ( & self ) -> Result < State , Error > {
2023-02-25 17:30:26 +00:00
let internal_bin = gst ::Bin ::with_name ( " internal " ) ;
let transcription_bin = gst ::Bin ::with_name ( " transcription-bin " ) ;
2022-10-19 16:18:43 +00:00
let audio_tee = gst ::ElementFactory ::make ( " tee " )
// Protect passthrough enable (and resulting dynamic reconfigure)
// from non-streaming thread
. property ( " allow-not-linked " , true )
. build ( ) ? ;
let cccombiner = gst ::ElementFactory ::make ( " cccombiner " )
. name ( " cccombiner " )
. build ( ) ? ;
2023-07-24 13:51:07 +00:00
let transcriber_resample = gst ::ElementFactory ::make ( " audioresample " ) . build ( ) ? ;
2022-10-19 16:18:43 +00:00
let transcriber_aconv = gst ::ElementFactory ::make ( " audioconvert " ) . build ( ) ? ;
let transcriber = gst ::ElementFactory ::make ( " awstranscriber " )
. name ( " transcriber " )
2023-05-09 14:24:15 +00:00
. property (
" language-code " ,
& self . settings . lock ( ) . unwrap ( ) . language_code ,
)
2022-10-19 16:18:43 +00:00
. build ( ) ? ;
let audio_queue_passthrough = gst ::ElementFactory ::make ( " queue " ) . build ( ) ? ;
let video_queue = gst ::ElementFactory ::make ( " queue " ) . build ( ) ? ;
let cccapsfilter = gst ::ElementFactory ::make ( " capsfilter " ) . build ( ) ? ;
let transcription_valve = gst ::ElementFactory ::make ( " valve " )
. property_from_str ( " drop-mode " , " transform-to-gap " )
. build ( ) ? ;
2023-11-21 13:09:22 +00:00
let settings = self . settings . lock ( ) . unwrap ( ) ;
let mux_method = settings . mux_method ;
let ccmux = match mux_method {
MuxMethod ::Cea608 = > gst ::ElementFactory ::make ( " cea608mux " )
. property_from_str ( " start-time-selection " , " first " )
. build ( ) ? ,
MuxMethod ::Cea708 = > gst ::ElementFactory ::make ( " cea708mux " )
. property_from_str ( " start-time-selection " , " first " )
. build ( ) ? ,
} ;
2023-04-07 15:46:45 +00:00
let ccmux_filter = gst ::ElementFactory ::make ( " capsfilter " ) . build ( ) ? ;
2021-06-22 20:08:08 +00:00
2023-03-24 23:14:46 +00:00
let mut transcription_channels = HashMap ::new ( ) ;
2023-11-21 13:09:22 +00:00
self . construct_transcription_channels (
& settings ,
settings . mux_method ,
& mut transcription_channels ,
) ? ;
drop ( settings ) ;
2023-03-24 23:14:46 +00:00
2021-06-22 20:08:08 +00:00
Ok ( State {
2023-11-21 13:09:22 +00:00
mux_method ,
2021-06-22 20:08:08 +00:00
framerate : None ,
internal_bin ,
audio_queue_passthrough ,
video_queue ,
2023-07-24 13:51:07 +00:00
transcriber_resample ,
2021-06-22 20:08:08 +00:00
transcriber_aconv ,
transcriber ,
2023-04-04 16:25:40 +00:00
ccmux ,
2023-04-07 15:46:45 +00:00
ccmux_filter ,
2021-06-22 20:08:08 +00:00
audio_tee ,
cccombiner ,
transcription_bin ,
2023-03-24 23:14:46 +00:00
transcription_channels ,
2021-06-22 20:08:08 +00:00
cccapsfilter ,
2022-03-07 16:06:12 +00:00
transcription_valve ,
2021-06-22 20:08:08 +00:00
tearing_down : false ,
} )
}
2021-09-08 12:33:31 +00:00
#[ allow(clippy::single_match) ]
2022-10-09 13:06:59 +00:00
fn video_sink_event ( & self , pad : & gst ::Pad , event : gst ::Event ) -> bool {
2021-06-22 20:08:08 +00:00
use gst ::EventView ;
2022-02-21 17:43:46 +00:00
gst ::log! ( CAT , obj : pad , " Handling event {:?} " , event ) ;
2021-06-22 20:08:08 +00:00
match event . view ( ) {
EventView ::Caps ( e ) = > {
let mut state = self . state . lock ( ) . unwrap ( ) ;
if let Some ( ref mut state ) = state . as_mut ( ) {
let caps = e . caps ( ) ;
let s = caps . structure ( 0 ) . unwrap ( ) ;
let had_framerate = state . framerate . is_some ( ) ;
if let Ok ( framerate ) = s . get ::< gst ::Fraction > ( " framerate " ) {
state . framerate = Some ( framerate ) ;
} else {
state . framerate = Some ( gst ::Fraction ::new ( 30 , 1 ) ) ;
}
if ! had_framerate {
2022-02-21 17:43:46 +00:00
gst ::info! (
2021-06-22 20:08:08 +00:00
CAT ,
2022-10-09 13:06:59 +00:00
imp : self ,
2021-06-22 20:08:08 +00:00
" Received video caps, setting up transcription "
) ;
2022-10-09 13:06:59 +00:00
self . setup_transcription ( state ) ;
2021-06-22 20:08:08 +00:00
}
}
2022-10-23 20:03:22 +00:00
gst ::Pad ::event_default ( pad , Some ( & * self . obj ( ) ) , event )
2021-06-22 20:08:08 +00:00
}
2022-10-23 20:03:22 +00:00
_ = > gst ::Pad ::event_default ( pad , Some ( & * self . obj ( ) ) , event ) ,
2021-06-22 20:08:08 +00:00
}
}
}
#[ glib::object_subclass ]
impl ObjectSubclass for TranscriberBin {
2022-10-23 15:42:58 +00:00
const NAME : & 'static str = " GstTranscriberBin " ;
2021-06-22 20:08:08 +00:00
type Type = super ::TranscriberBin ;
type ParentType = gst ::Bin ;
fn with_class ( klass : & Self ::Class ) -> Self {
let templ = klass . pad_template ( " sink_audio " ) . unwrap ( ) ;
2023-05-10 15:02:08 +00:00
let audio_sinkpad = gst ::GhostPad ::from_template ( & templ ) ;
2021-06-22 20:08:08 +00:00
let templ = klass . pad_template ( " src_audio " ) . unwrap ( ) ;
2023-05-10 15:02:08 +00:00
let audio_srcpad = gst ::GhostPad ::builder_from_template ( & templ )
2021-06-22 20:08:08 +00:00
. query_function ( | pad , parent , query | {
TranscriberBin ::catch_panic_pad_function (
parent ,
| | false ,
2022-10-09 13:06:59 +00:00
| transcriber | transcriber . src_query ( pad . upcast_ref ( ) , query ) ,
2021-06-22 20:08:08 +00:00
)
} )
. build ( ) ;
let templ = klass . pad_template ( " sink_video " ) . unwrap ( ) ;
2023-05-10 15:02:08 +00:00
let video_sinkpad = gst ::GhostPad ::builder_from_template ( & templ )
2021-06-22 20:08:08 +00:00
. event_function ( | pad , parent , event | {
TranscriberBin ::catch_panic_pad_function (
parent ,
| | false ,
2022-10-09 13:06:59 +00:00
| transcriber | transcriber . video_sink_event ( pad . upcast_ref ( ) , event ) ,
2021-06-22 20:08:08 +00:00
)
} )
. build ( ) ;
let templ = klass . pad_template ( " src_video " ) . unwrap ( ) ;
2023-05-10 15:02:08 +00:00
let video_srcpad = gst ::GhostPad ::builder_from_template ( & templ )
2021-06-22 20:08:08 +00:00
. query_function ( | pad , parent , query | {
TranscriberBin ::catch_panic_pad_function (
parent ,
| | false ,
2022-10-09 13:06:59 +00:00
| transcriber | transcriber . src_query ( pad . upcast_ref ( ) , query ) ,
2021-06-22 20:08:08 +00:00
)
} )
. build ( ) ;
Self {
audio_srcpad ,
video_srcpad ,
audio_sinkpad ,
video_sinkpad ,
state : Mutex ::new ( None ) ,
settings : Mutex ::new ( Settings ::default ( ) ) ,
}
}
}
impl ObjectImpl for TranscriberBin {
fn properties ( ) -> & 'static [ glib ::ParamSpec ] {
static PROPERTIES : Lazy < Vec < glib ::ParamSpec > > = Lazy ::new ( | | {
vec! [
2022-08-18 12:04:15 +00:00
glib ::ParamSpecBoolean ::builder ( " passthrough " )
. nick ( " Passthrough " )
. blurb ( " Whether transcription should occur " )
. default_value ( DEFAULT_PASSTHROUGH )
. mutable_playing ( )
. build ( ) ,
glib ::ParamSpecUInt ::builder ( " latency " )
. nick ( " Latency " )
. blurb ( " Amount of milliseconds to allow the transcriber " )
. default_value ( DEFAULT_LATENCY . mseconds ( ) as u32 )
. mutable_ready ( )
. build ( ) ,
glib ::ParamSpecUInt ::builder ( " accumulate-time " )
. nick ( " accumulate-time " )
. blurb ( " Cut-off time for textwrap accumulation, in milliseconds (0=do not accumulate). \
Set this to a non - default value if you plan to switch to pop - on mode " )
. default_value ( DEFAULT_ACCUMULATE . mseconds ( ) as u32 )
. mutable_ready ( )
. build ( ) ,
2023-01-21 16:13:48 +00:00
glib ::ParamSpecEnum ::builder_with_default ( " mode " , DEFAULT_MODE )
2022-08-18 12:04:15 +00:00
. nick ( " Mode " )
. blurb ( " Which closed caption mode to operate in " )
. mutable_playing ( )
. build ( ) ,
2022-09-05 08:45:47 +00:00
glib ::ParamSpecBoxed ::builder ::< gst ::Caps > ( " cc-caps " )
2022-08-18 12:04:15 +00:00
. nick ( " Closed Caption caps " )
. blurb ( " The expected format of the closed captions " )
. mutable_ready ( )
. build ( ) ,
2022-09-05 08:45:47 +00:00
glib ::ParamSpecObject ::builder ::< gst ::Element > ( " transcriber " )
2022-08-18 12:04:15 +00:00
. nick ( " Transcriber " )
. blurb ( " The transcriber element to use " )
. mutable_ready ( )
. build ( ) ,
2023-01-21 16:13:48 +00:00
glib ::ParamSpecEnum ::builder_with_default ( " caption-source " , DEFAULT_CAPTION_SOURCE )
2022-08-18 12:04:15 +00:00
. nick ( " Caption source " )
. blurb ( " Caption source to use. \
2022-03-07 16:06:12 +00:00
If \ " Transcription \" or \" Inband \" is selected, the caption meta \
2022-08-18 12:04:15 +00:00
of the other source will be dropped by transcriberbin " )
. mutable_playing ( )
. build ( ) ,
2023-03-24 23:14:46 +00:00
glib ::ParamSpecBoxed ::builder ::< gst ::Structure > ( " translation-languages " )
. nick ( " Translation languages " )
2023-11-21 13:09:22 +00:00
. blurb ( " A map of language codes to caption channels, e.g. translation-languages= \" languages, transcript={CC1, 708_1}, fr={708_2, CC3} \" will map the French translation to CC1/service 1 and the original transcript to CC3/service 2 " )
2023-04-04 16:25:40 +00:00
. mutable_playing ( )
2023-03-24 23:14:46 +00:00
. build ( ) ,
glib ::ParamSpecUInt ::builder ( " translate-latency " )
. nick ( " Translation Latency " )
. blurb ( " Amount of extra milliseconds to allow for translating " )
. default_value ( DEFAULT_TRANSLATE_LATENCY . mseconds ( ) as u32 )
. mutable_ready ( )
. build ( ) ,
2023-05-09 14:24:15 +00:00
glib ::ParamSpecString ::builder ( " language-code " )
. nick ( " Language Code " )
. blurb ( " The language of the input stream " )
. default_value ( Some ( DEFAULT_INPUT_LANG_CODE ) )
. mutable_playing ( )
. build ( ) ,
2023-11-21 13:09:22 +00:00
glib ::ParamSpecEnum ::builder ( " mux-method " )
. nick ( " Mux Method " )
. blurb ( " The method for muxing multiple transcription streams " )
. default_value ( DEFAULT_MUX_METHOD )
. construct ( )
. build ( ) ,
2021-06-22 20:08:08 +00:00
]
} ) ;
PROPERTIES . as_ref ( )
}
2022-10-09 13:06:59 +00:00
fn set_property ( & self , _id : usize , value : & glib ::Value , pspec : & glib ::ParamSpec ) {
2021-06-22 20:08:08 +00:00
match pspec . name ( ) {
" passthrough " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
let old_passthrough = settings . passthrough ;
let new_passthrough = value . get ( ) . expect ( " type checked upstream " ) ;
settings . passthrough = new_passthrough ;
if old_passthrough ! = new_passthrough {
drop ( settings ) ;
2022-10-09 13:06:59 +00:00
self . block_and_update ( new_passthrough ) ;
2021-06-22 20:08:08 +00:00
}
}
" latency " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
settings . latency = gst ::ClockTime ::from_mseconds (
value . get ::< u32 > ( ) . expect ( " type checked upstream " ) . into ( ) ,
) ;
}
" accumulate-time " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
settings . accumulate_time = gst ::ClockTime ::from_mseconds (
value . get ::< u32 > ( ) . expect ( " type checked upstream " ) . into ( ) ,
) ;
}
" mode " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
let old_mode = settings . mode ;
let new_mode = value . get ( ) . expect ( " type checked upstream " ) ;
settings . mode = new_mode ;
if old_mode ! = new_mode {
drop ( settings ) ;
2022-10-09 13:06:59 +00:00
self . setup_cc_mode ( self . state . lock ( ) . unwrap ( ) . as_ref ( ) . unwrap ( ) ) ;
2021-06-22 20:08:08 +00:00
}
}
" cc-caps " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
settings . cc_caps = value . get ( ) . expect ( " type checked upstream " ) ;
}
" transcriber " = > {
let mut s = self . state . lock ( ) . unwrap ( ) ;
if let Some ( ref mut state ) = s . as_mut ( ) {
let old_transcriber = state . transcriber . clone ( ) ;
state . transcriber = value . get ( ) . expect ( " type checked upstream " ) ;
if old_transcriber ! = state . transcriber {
2022-10-09 13:06:59 +00:00
match self . relink_transcriber ( state , & old_transcriber ) {
2021-06-22 20:08:08 +00:00
Ok ( ( ) ) = > ( ) ,
Err ( err ) = > {
2022-02-21 17:43:46 +00:00
gst ::error! ( CAT , " invalid transcriber: {} " , err ) ;
2021-06-22 20:08:08 +00:00
drop ( s ) ;
* self . state . lock ( ) . unwrap ( ) = None ;
}
}
}
}
}
2022-03-07 16:06:12 +00:00
" caption-source " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
settings . caption_source = value . get ( ) . expect ( " type checked upstream " ) ;
let s = self . state . lock ( ) . unwrap ( ) ;
if let Some ( state ) = s . as_ref ( ) {
if settings . caption_source = = CaptionSource ::Inband {
2022-10-09 13:06:59 +00:00
gst ::debug! ( CAT , imp : self , " Use inband caption, dropping transcription " ) ;
2022-03-07 16:06:12 +00:00
state . transcription_valve . set_property ( " drop " , true ) ;
} else {
2022-10-09 13:06:59 +00:00
gst ::debug! ( CAT , imp : self , " Stop dropping transcription " ) ;
2022-03-07 16:06:12 +00:00
state . transcription_valve . set_property ( " drop " , false ) ;
}
}
}
2023-03-24 23:14:46 +00:00
" translation-languages " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
settings . translation_languages = value
. get ::< Option < gst ::Structure > > ( )
2023-04-04 16:25:40 +00:00
. expect ( " type checked upstream " ) ;
gst ::debug! (
CAT ,
imp : self ,
" Updated translation-languages {:?} " ,
settings . translation_languages
) ;
drop ( settings ) ;
2023-05-09 14:24:15 +00:00
self . update_languages ( false ) ;
2023-03-24 23:14:46 +00:00
}
" translate-latency " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
settings . translate_latency = gst ::ClockTime ::from_mseconds (
value . get ::< u32 > ( ) . expect ( " type checked upstream " ) . into ( ) ,
) ;
}
2023-05-09 14:24:15 +00:00
" language-code " = > {
let code = value
. get ::< Option < String > > ( )
. expect ( " type checked upstream " )
. unwrap_or_else ( | | String ::from ( DEFAULT_INPUT_LANG_CODE ) ) ;
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
if settings . language_code ! = code {
gst ::debug! (
CAT ,
imp : self ,
" Updating language code {} -> {} " ,
settings . language_code ,
code
) ;
settings . language_code = code ;
drop ( settings ) ;
self . update_languages ( true )
}
}
2023-11-21 13:09:22 +00:00
" mux-method " = > {
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
settings . mux_method = value . get ( ) . expect ( " type checked upstream " )
}
2021-06-22 20:08:08 +00:00
_ = > unimplemented! ( ) ,
}
}
2022-10-09 13:06:59 +00:00
fn property ( & self , _id : usize , pspec : & glib ::ParamSpec ) -> glib ::Value {
2021-06-22 20:08:08 +00:00
match pspec . name ( ) {
" passthrough " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
settings . passthrough . to_value ( )
}
" latency " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
( settings . latency . mseconds ( ) as u32 ) . to_value ( )
}
" accumulate-time " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
( settings . accumulate_time . mseconds ( ) as u32 ) . to_value ( )
}
" mode " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
settings . mode . to_value ( )
}
" cc-caps " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
settings . cc_caps . to_value ( )
}
" transcriber " = > {
let state = self . state . lock ( ) . unwrap ( ) ;
2021-07-30 10:53:35 +00:00
if let Some ( state ) = state . as_ref ( ) {
2021-06-22 20:08:08 +00:00
state . transcriber . to_value ( )
} else {
let ret : Option < gst ::Element > = None ;
ret . to_value ( )
}
}
2022-03-07 16:06:12 +00:00
" caption-source " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
settings . caption_source . to_value ( )
}
2023-03-24 23:14:46 +00:00
" translation-languages " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
settings . translation_languages . to_value ( )
}
" translate-latency " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
( settings . translate_latency . mseconds ( ) as u32 ) . to_value ( )
}
2023-05-09 14:24:15 +00:00
" language-code " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
settings . language_code . to_value ( )
}
2023-11-21 13:09:22 +00:00
" mux-method " = > {
let settings = self . settings . lock ( ) . unwrap ( ) ;
settings . mux_method . to_value ( )
}
2021-06-22 20:08:08 +00:00
_ = > unimplemented! ( ) ,
}
}
2022-10-09 13:06:59 +00:00
fn constructed ( & self ) {
self . parent_constructed ( ) ;
2021-06-22 20:08:08 +00:00
2022-10-23 20:03:22 +00:00
let obj = self . obj ( ) ;
2021-06-22 20:08:08 +00:00
obj . add_pad ( & self . audio_srcpad ) . unwrap ( ) ;
obj . add_pad ( & self . audio_sinkpad ) . unwrap ( ) ;
obj . add_pad ( & self . video_srcpad ) . unwrap ( ) ;
obj . add_pad ( & self . video_sinkpad ) . unwrap ( ) ;
* self . state . lock ( ) . unwrap ( ) = match self . build_state ( ) {
2022-10-09 13:06:59 +00:00
Ok ( mut state ) = > match self . construct_internal_bin ( & mut state ) {
2021-06-22 20:08:08 +00:00
Ok ( ( ) ) = > Some ( state ) ,
Err ( err ) = > {
2022-02-21 17:43:46 +00:00
gst ::error! ( CAT , " Failed to build internal bin: {} " , err ) ;
2021-06-22 20:08:08 +00:00
None
}
} ,
Err ( err ) = > {
2022-02-21 17:43:46 +00:00
gst ::error! ( CAT , " Failed to build state: {} " , err ) ;
2021-06-22 20:08:08 +00:00
None
}
}
}
}
2021-10-23 08:57:31 +00:00
impl GstObjectImpl for TranscriberBin { }
2021-06-22 20:08:08 +00:00
impl ElementImpl for TranscriberBin {
fn metadata ( ) -> Option < & 'static gst ::subclass ::ElementMetadata > {
static ELEMENT_METADATA : Lazy < gst ::subclass ::ElementMetadata > = Lazy ::new ( | | {
gst ::subclass ::ElementMetadata ::new (
" TranscriberBin " ,
" Audio / Video / Text " ,
" Transcribes audio and adds it as closed captions " ,
" Mathieu Duponchelle <mathieu@centricular.com> " ,
)
} ) ;
Some ( & * ELEMENT_METADATA )
}
fn pad_templates ( ) -> & 'static [ gst ::PadTemplate ] {
static PAD_TEMPLATES : Lazy < Vec < gst ::PadTemplate > > = Lazy ::new ( | | {
2023-04-07 16:51:12 +00:00
let caps = gst ::Caps ::builder ( " video/x-raw " ) . any_features ( ) . build ( ) ;
2021-06-22 20:08:08 +00:00
let video_src_pad_template = gst ::PadTemplate ::new (
" src_video " ,
gst ::PadDirection ::Src ,
gst ::PadPresence ::Always ,
& caps ,
)
. unwrap ( ) ;
let video_sink_pad_template = gst ::PadTemplate ::new (
" sink_video " ,
gst ::PadDirection ::Sink ,
gst ::PadPresence ::Always ,
& caps ,
)
. unwrap ( ) ;
2021-11-06 07:34:10 +00:00
let caps = gst ::Caps ::builder ( " audio/x-raw " ) . build ( ) ;
2021-06-22 20:08:08 +00:00
let audio_src_pad_template = gst ::PadTemplate ::new (
" src_audio " ,
gst ::PadDirection ::Src ,
gst ::PadPresence ::Always ,
& caps ,
)
. unwrap ( ) ;
let audio_sink_pad_template = gst ::PadTemplate ::new (
" sink_audio " ,
gst ::PadDirection ::Sink ,
gst ::PadPresence ::Always ,
& caps ,
)
. unwrap ( ) ;
vec! [
video_src_pad_template ,
video_sink_pad_template ,
audio_src_pad_template ,
audio_sink_pad_template ,
]
} ) ;
PAD_TEMPLATES . as_ref ( )
}
2021-09-08 12:33:31 +00:00
#[ allow(clippy::single_match) ]
2021-06-22 20:08:08 +00:00
fn change_state (
& self ,
transition : gst ::StateChange ,
) -> Result < gst ::StateChangeSuccess , gst ::StateChangeError > {
2022-10-09 13:06:59 +00:00
gst ::trace! ( CAT , imp : self , " Changing state {:?} " , transition ) ;
2021-06-22 20:08:08 +00:00
match transition {
gst ::StateChange ::ReadyToPaused = > {
let mut state = self . state . lock ( ) . unwrap ( ) ;
if let Some ( ref mut state ) = state . as_mut ( ) {
if state . framerate . is_some ( ) {
2022-02-21 17:43:46 +00:00
gst ::info! (
2021-06-22 20:08:08 +00:00
CAT ,
2022-10-09 13:06:59 +00:00
imp : self ,
2021-06-22 20:08:08 +00:00
" Received video caps, setting up transcription "
) ;
2022-10-09 13:06:59 +00:00
self . setup_transcription ( state ) ;
2021-06-22 20:08:08 +00:00
}
} else {
2023-03-24 22:35:10 +00:00
drop ( state ) ;
2022-10-09 13:06:59 +00:00
gst ::element_imp_error! (
self ,
2021-06-22 20:08:08 +00:00
gst ::StreamError ::Failed ,
[ " Can't change state with no state " ]
) ;
return Err ( gst ::StateChangeError ) ;
}
}
_ = > ( ) ,
}
2022-10-09 13:06:59 +00:00
self . parent_change_state ( transition )
2021-06-22 20:08:08 +00:00
}
}
2021-09-27 23:50:49 +00:00
impl BinImpl for TranscriberBin {
2022-10-09 13:06:59 +00:00
fn handle_message ( & self , msg : gst ::Message ) {
2021-09-27 23:50:49 +00:00
use gst ::MessageView ;
match msg . view ( ) {
2022-01-19 13:07:45 +00:00
MessageView ::Error ( m ) = > {
2021-09-27 23:50:49 +00:00
/* We must have a state here */
let s = self . state . lock ( ) . unwrap ( ) ;
if let Some ( state ) = s . as_ref ( ) {
2023-01-05 10:32:01 +00:00
if msg . src ( ) = = Some ( state . transcriber . upcast_ref ( ) ) {
2022-02-21 17:43:46 +00:00
gst ::error! (
2021-09-27 23:50:49 +00:00
CAT ,
2022-10-09 13:06:59 +00:00
imp : self ,
2021-09-27 23:50:49 +00:00
" Transcriber has posted an error ({:?}), going back to passthrough " ,
m
) ;
drop ( s ) ;
let mut settings = self . settings . lock ( ) . unwrap ( ) ;
settings . passthrough = true ;
drop ( settings ) ;
2022-10-23 20:03:22 +00:00
self . obj ( ) . notify ( " passthrough " ) ;
self . obj ( ) . call_async ( move | bin | {
2022-01-17 17:36:41 +00:00
let thiz = bin . imp ( ) ;
2022-10-09 13:06:59 +00:00
thiz . block_and_update ( true ) ;
2021-09-27 23:50:49 +00:00
} ) ;
} else {
drop ( s ) ;
2022-10-09 13:06:59 +00:00
self . parent_handle_message ( msg ) ;
2021-09-27 23:50:49 +00:00
}
} else {
drop ( s ) ;
2022-10-09 13:06:59 +00:00
self . parent_handle_message ( msg ) ;
2021-09-27 23:50:49 +00:00
}
}
2022-10-09 13:06:59 +00:00
_ = > self . parent_handle_message ( msg ) ,
2021-09-27 23:50:49 +00:00
}
}
}