// Copyright (C) 2023 Matthew Waters // // This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. // If a copy of the MPL was not distributed with this file, You can obtain one at // . // // SPDX-License-Identifier: MPL-2.0 use cea708_types::{tables::*, DTVCCPacket}; use gst::glib; use gst::prelude::*; use gst::subclass::prelude::*; use atomic_refcell::AtomicRefCell; use crate::cea608tocea708::fmt::Cea708ServiceWriter; use crate::cea608utils::*; use once_cell::sync::Lazy; #[derive(Debug, Copy, Clone)] enum Cea608Format { S334_1A, RawField0, RawField1, } struct Cea608State { tracker: [Cea608StateTracker; 2], format: Cea608Format, service: [Cea608ServiceState; 4], } impl Default for Cea608State { fn default() -> Self { Self { tracker: [Cea608StateTracker::default(), Cea608StateTracker::default()], format: Cea608Format::RawField0, service: [ Cea608ServiceState::default(), Cea608ServiceState::default(), Cea608ServiceState::default(), Cea608ServiceState::default(), ], } } } struct Cea608ServiceState { mode: Option, base_row: u8, } impl Default for Cea608ServiceState { fn default() -> Self { Self { mode: None, base_row: 15, } } } struct Cea708ServiceState { writer: Cea708ServiceWriter, pen_location: SetPenLocationArgs, pen_color: SetPenColorArgs, pen_attributes: SetPenAttributesArgs, } fn textstyle_foreground_color(style: TextStyle) -> Color { match style { TextStyle::Red => Color { r: ColorValue::Full, g: ColorValue::None, b: ColorValue::None, }, TextStyle::Green => Color { r: ColorValue::None, g: ColorValue::Full, b: ColorValue::None, }, TextStyle::Blue => Color { r: ColorValue::None, g: ColorValue::None, b: ColorValue::Full, }, TextStyle::Cyan => Color { r: ColorValue::None, g: ColorValue::Full, b: ColorValue::Full, }, TextStyle::Yellow => Color { r: ColorValue::Full, g: ColorValue::Full, b: ColorValue::None, }, TextStyle::Magenta => Color { r: ColorValue::Full, g: ColorValue::None, b: ColorValue::Full, }, TextStyle::White | TextStyle::ItalicWhite => Color { r: ColorValue::Full, g: ColorValue::Full, b: ColorValue::Full, }, } } fn textstyle_to_pen_color(style: TextStyle) -> SetPenColorArgs { let black = Color { r: ColorValue::None, g: ColorValue::None, b: ColorValue::None, }; SetPenColorArgs { foreground_color: textstyle_foreground_color(style), foreground_opacity: Opacity::Solid, background_color: black, background_opacity: Opacity::Solid, edge_color: black, } } fn textstyle_is_italics(style: TextStyle) -> bool { style == TextStyle::ItalicWhite } fn cea608_mode_visible_rows(mode: Cea608Mode) -> u8 { match mode { Cea608Mode::RollUp2 => 2, Cea608Mode::RollUp3 => 3, Cea608Mode::RollUp4 => 4, _ => unreachable!(), } } impl Cea708ServiceState { fn new(service_no: u8) -> Self { Self { writer: Cea708ServiceWriter::new(service_no), pen_location: SetPenLocationArgs::new(0, 0), pen_color: textstyle_to_pen_color(TextStyle::White), pen_attributes: SetPenAttributesArgs::new( PenSize::Standard, FontStyle::Default, TextTag::Dialog, TextOffset::Normal, false, false, EdgeType::None, ), } } fn new_mode(&mut self, cea608_mode: Cea608Mode, base_row: u8) { let new_row = if cea608_mode.is_rollup() { cea608_mode_visible_rows(cea608_mode) - 1 } else { 0 }; match cea608_mode { Cea608Mode::PopOn => self.writer.popon_preamble().unwrap(), Cea608Mode::PaintOn => self.writer.paint_on_preamble().unwrap(), Cea608Mode::RollUp2 => self.writer.rollup_preamble(2, base_row).unwrap(), Cea608Mode::RollUp3 => self.writer.rollup_preamble(3, base_row).unwrap(), Cea608Mode::RollUp4 => self.writer.rollup_preamble(4, base_row).unwrap(), }; // we have redefined then window so all the attributes have been reset self.pen_location.row = new_row; self.pen_location.column = 0; self.pen_color = textstyle_to_pen_color(TextStyle::White); self.pen_attributes.underline = false; self.pen_attributes.italics = false; } fn handle_text(&mut self, text: Cea608Text) { if text.code_space == CodeSpace::WestEU { self.writer.push_codes(&[Code::BS]).unwrap(); } if let Some(c) = text.char1 { if self.pen_location.column > 31 { self.writer.push_codes(&[Code::BS]).unwrap(); } self.writer.write_char(c).unwrap(); self.pen_location.column = std::cmp::min(self.pen_location.column + 1, 32); } if let Some(c) = text.char2 { if self.pen_location.column > 31 { self.writer.push_codes(&[Code::BS]).unwrap(); } self.writer.write_char(c).unwrap(); self.pen_location.column = std::cmp::min(self.pen_location.column + 1, 32); } } fn handle_preamble(&mut self, preamble: Preamble) { let mut need_pen_location = false; // TODO: may need a better algorithm then compressing the top four rows let new_row = std::cmp::max(0, preamble.row) as u8; if self.pen_location.row != new_row { need_pen_location = true; self.pen_location.row = new_row; } if self.pen_location.column != preamble.col as u8 { need_pen_location = true; self.pen_location.column = preamble.col as u8; } if need_pen_location { self.writer.set_pen_location(self.pen_location).unwrap(); } let mut need_pen_attributes = false; if self.pen_attributes.italics != textstyle_is_italics(preamble.style) { need_pen_attributes = true; self.pen_attributes.italics = textstyle_is_italics(preamble.style); } if self.pen_attributes.underline != (preamble.underline > 0) { need_pen_attributes = true; self.pen_attributes.underline = preamble.underline > 0; } if need_pen_attributes { self.writer.set_pen_attributes(self.pen_attributes).unwrap(); } if self.pen_color.foreground_color != textstyle_foreground_color(preamble.style) { self.pen_color.foreground_color = textstyle_foreground_color(preamble.style); self.writer.set_pen_color(self.pen_color).unwrap(); } } fn handle_midrowchange(&mut self, midrowchange: MidRowChange) { self.writer.write_char(' ').unwrap(); if self.pen_color.foreground_color != textstyle_foreground_color(midrowchange.style) { self.pen_color.foreground_color = textstyle_foreground_color(midrowchange.style); self.writer.set_pen_color(self.pen_color).unwrap(); } let mut need_pen_attributes = false; if self.pen_attributes.italics != textstyle_is_italics(midrowchange.style) { need_pen_attributes = true; self.pen_attributes.italics = textstyle_is_italics(midrowchange.style); } if self.pen_attributes.underline != midrowchange.underline { need_pen_attributes = true; self.pen_attributes.underline = midrowchange.underline; } if need_pen_attributes { self.writer.set_pen_attributes(self.pen_attributes).unwrap(); } } fn carriage_return(&mut self) { self.writer.carriage_return().unwrap(); } } struct Cea708State { packet_counter: u8, service_state: [Cea708ServiceState; 4], } impl Cea708State { fn take_buffer(&mut self, s334_1a_data: &[u8]) -> Option { let mut packet = DTVCCPacket::new(self.packet_counter); self.packet_counter += 1; self.packet_counter &= 0x3; for state in self.service_state.iter_mut() { if let Some(service) = state.writer.take_service() { if let Err(e) = packet.push_service(service.clone()) { gst::warning!( CAT, "failed to add service:{} to outgoing packet: {:?}", service.number(), e ); } } } let mut cc_data = Vec::with_capacity(64); assert!(s334_1a_data.len() % 3 == 0); for triple in s334_1a_data.chunks(3) { if (triple[0] & 0x80) > 0 { cc_data.push(0xfd); } else { cc_data.push(0xfc); } cc_data.extend(&triple[1..]); } let mut ccp_data = Vec::with_capacity(128); packet.write_cc_data(&mut ccp_data).ok()?; gst::trace!( CAT, "take_buffer produced cc_data_len:{} cc_data:{cc_data:?}", cc_data.len() ); // ignore the 2 byte cc_data header that is unused in GStreamer if ccp_data.len() > 2 { cc_data.extend(&ccp_data[2..]); } else if cc_data.is_empty() { return None; } Some(gst::Buffer::from_slice(cc_data)) } } struct State { cea608: Cea608State, cea708: Cea708State, } impl Default for State { fn default() -> Self { State { cea608: Cea608State::default(), cea708: Cea708State { packet_counter: 0, service_state: [ Cea708ServiceState::new(1), Cea708ServiceState::new(2), Cea708ServiceState::new(3), Cea708ServiceState::new(4), ], }, } } } enum BufferOrEvent { Buffer(gst::Buffer), Event(gst::Event), } impl State { fn field_channel_to_index(&self, field: u8, channel: i32) -> usize { match (field, channel) { (0, 0 | 2) => 0, (0, 1 | 3) => 2, (1, 0 | 2) => 1, (1, 1 | 3) => 3, _ => unreachable!(), } } fn service_state_from_608_field_channel( &mut self, field: u8, channel: i32, ) -> &mut Cea708ServiceState { &mut self.cea708.service_state[self.field_channel_to_index(field, channel)] } fn new_mode(&mut self, field: u8, channel: i32, cea608_mode: Cea608Mode) { let idx = self.field_channel_to_index(field, channel); if let Some(old_mode) = self.cea608.service[idx].mode { if cea608_mode.is_rollup() && matches!(old_mode, Cea608Mode::PopOn | Cea608Mode::PaintOn) { // https://www.law.cornell.edu/cfr/text/47/79.101 (f)(1)(x) gst::trace!(CAT, "change to rollup from pop/paint-on"); self.cea708.service_state[idx] .writer .clear_hidden_window() .unwrap(); self.cea708.service_state[idx] .writer .clear_current_window() .unwrap(); self.cea608.service[idx].base_row = 15; } if old_mode.is_rollup() && cea608_mode.is_rollup() { // https://www.law.cornell.edu/cfr/text/47/79.101 (f)(1)(x) let old_count = cea608_mode_visible_rows(old_mode); let new_count = cea608_mode_visible_rows(cea608_mode); gst::trace!( CAT, "change of rollup row count from {old_count} to {new_count}", ); if old_count > new_count { // push the captions on the top of the window before we shrink the size of the // window for _ in new_count..old_count { self.cea708.service_state[idx] .writer .push_codes(&[Code::CR]) .unwrap(); } } } } self.cea608.service[idx].mode = Some(cea608_mode); let base_row = if cea608_mode.is_rollup() { self.cea608.service[idx].base_row } else { 0 }; self.cea708.service_state[idx].new_mode(cea608_mode, base_row); } fn handle_cc_data(&mut self, imp: &Cea608ToCea708, field: u8, cc_data: u16) { self.cea608.tracker[field as usize].push_cc_data(cc_data); let mut channel = None; if let Some(cea608) = self.cea608.tracker[field as usize].pop() { gst::trace!( CAT, imp: imp, "have field:{field} channel:{} {cea608:?}", cea608.channel() ); if !matches!(cea608, Cea608::Duplicate) { channel = Some(cea608.channel()); } match cea608 { Cea608::Duplicate => (), Cea608::NewMode(chan, new_mode) => { self.new_mode(field, chan, new_mode); } Cea608::Text(text) => { let state = self.service_state_from_608_field_channel(field, text.chan); state.handle_text(text); } Cea608::EndOfCaption(chan) => { let state = self.service_state_from_608_field_channel(field, chan); state.writer.end_of_caption().unwrap(); state.writer.etx().unwrap(); } Cea608::Preamble(mut preamble) => { let idx = self.field_channel_to_index(field, preamble.chan); let rollup_count = self.cea608.service[idx] .mode .map(|mode| { if mode.is_rollup() { cea608_mode_visible_rows(mode) } else { 0 } }) .unwrap_or(0); if rollup_count > 0 { // https://www.law.cornell.edu/cfr/text/47/79.101 (f)(1)(ii) let old_base_row = self.cea608.service[idx].base_row; self.cea608.service[idx].base_row = preamble.row as u8; let state = self.service_state_from_608_field_channel(field, preamble.chan); if old_base_row != preamble.row as u8 { state .writer .rollup_preamble(rollup_count, preamble.row as u8) .unwrap(); } state.pen_location.row = rollup_count - 1; preamble.row = rollup_count as i32 - 1; } let state = self.service_state_from_608_field_channel(field, preamble.chan); state.handle_preamble(preamble); } Cea608::MidRowChange(midrowchange) => { let state = self.service_state_from_608_field_channel(field, midrowchange.chan); state.handle_midrowchange(midrowchange); } Cea608::Backspace(chan) => { let state = self.service_state_from_608_field_channel(field, chan); // TODO: handle removing a midrowchange state.pen_location.column = std::cmp::max(state.pen_location.column - 1, 0); state .writer .push_codes(&[cea708_types::tables::Code::BS]) .unwrap(); } Cea608::CarriageReturn(chan) => { if let Some(mode) = self.cea608.service[self.field_channel_to_index(field, chan)].mode { if mode.is_rollup() { let state = self.service_state_from_608_field_channel(field, chan); state.carriage_return(); } } } Cea608::EraseDisplay(chan) => { let state = self.service_state_from_608_field_channel(field, chan); state.writer.clear_current_window().unwrap(); } Cea608::EraseNonDisplay(chan) => { let state = self.service_state_from_608_field_channel(field, chan); state.writer.clear_hidden_window().unwrap(); } Cea608::TabOffset(chan, count) => { let state = self.service_state_from_608_field_channel(field, chan); state.pen_location.column = std::cmp::min(state.pen_location.column + count as u8, 32); } } if let Some(channel) = channel { let idx = self.field_channel_to_index(field, channel); if let Some( Cea608Mode::RollUp2 | Cea608Mode::RollUp3 | Cea608Mode::RollUp4 | Cea608Mode::PaintOn, ) = self.cea608.service[idx].mode { // FIXME: actually track state for when things have changed // and we need to send ETX self.cea708.service_state[idx] .writer .push_codes(&[cea708_types::tables::Code::ETX]) .unwrap(); } } } } fn take_buffer( &mut self, s334_1a_data: &[u8], pts: gst::ClockTime, duration: Option, ) -> BufferOrEvent { if let Some(mut buffer) = self.cea708.take_buffer(s334_1a_data) { { let buffer_ref = buffer.get_mut().unwrap(); buffer_ref.set_pts(Some(pts)); buffer_ref.set_duration(duration); } BufferOrEvent::Buffer(buffer) } else { BufferOrEvent::Event(gst::event::Gap::builder(pts).duration(duration).build()) } } } pub struct Cea608ToCea708 { srcpad: gst::Pad, sinkpad: gst::Pad, state: AtomicRefCell, } pub(crate) static CAT: Lazy = Lazy::new(|| { gst::DebugCategory::new( "cea608tocea708", gst::DebugColorFlags::empty(), Some("CEA-608 to CEA-708 Element"), ) }); impl Cea608ToCea708 { fn sink_chain( &self, pad: &gst::Pad, buffer: gst::Buffer, ) -> Result { gst::log!(CAT, obj: pad, "Handling buffer {:?}", buffer); let mut state = self.state.borrow_mut(); let buffer_pts = buffer.pts().ok_or_else(|| { gst::error!(CAT, obj: pad, "Require timestamped buffers"); gst::FlowError::Error })?; let data = buffer.map_readable().map_err(|_| { gst::error!(CAT, obj: pad, "Can't map buffer readable"); gst::FlowError::Error })?; let mut data_len = data.len(); let s334_1a_data = match state.cea608.format { Cea608Format::S334_1A => { if data_len % 3 != 0 { gst::warning!( CAT, obj: pad, "Invalid closed caption packet size, truncating" ); data_len -= data_len % 3; } if data_len < 3 { gst::warning!( CAT, obj: pad, "Invalid closed caption packet size, dropping" ); return Ok(gst::FlowSuccess::Ok); } for triple in data.chunks_exact(3) { let cc_data = (triple[1] as u16) << 8 | triple[2] as u16; let field = (triple[0] & 0x80) >> 7; state.handle_cc_data(self, field, cc_data); } data.to_vec() } Cea608Format::RawField0 | Cea608Format::RawField1 => { if data_len % 2 != 0 { gst::warning!( CAT, obj: pad, "Invalid closed caption packet size, truncating" ); data_len -= data_len % 3; } if data_len < 2 { gst::warning!( CAT, obj: pad, "Invalid closed caption packet size, dropping" ); return Ok(gst::FlowSuccess::Ok); } let field = match state.cea608.format { Cea608Format::RawField0 => 0, Cea608Format::RawField1 => 1, _ => unreachable!(), }; let mut s334_1a_data = Vec::with_capacity(data.len() / 2 * 3); for pair in data.chunks_exact(2) { let cc_data = (pair[0] as u16) << 8 | pair[1] as u16; state.handle_cc_data(self, field, cc_data); if field == 0 { s334_1a_data.push(0x00); } else { s334_1a_data.push(0x80); } s334_1a_data.push(pair[0]); s334_1a_data.push(pair[1]); } s334_1a_data } }; let buffer_or_event = state.take_buffer(&s334_1a_data, buffer_pts, buffer.duration()); drop(state); match buffer_or_event { BufferOrEvent::Buffer(buffer) => self.srcpad.push(buffer), BufferOrEvent::Event(event) => { self.srcpad.push_event(event); Ok(gst::FlowSuccess::Ok) } } } fn sink_event(&self, pad: &gst::Pad, event: gst::Event) -> bool { use gst::EventView; gst::log!(CAT, obj: pad, "Handling event {:?}", event); match event.view() { EventView::Caps(event) => { let mut state = self.state.borrow_mut(); let caps = event.caps(); let structure = caps.structure(0).expect("Caps has no structure"); let framerate = structure.get::("framerate").ok(); state.cea608.format = match structure.get::<&str>("format") { Ok("raw") => { if structure.has_field("field") { match structure.get("field") { Ok(0) => Cea608Format::RawField0, Ok(1) => Cea608Format::RawField1, _ => { gst::error!( CAT, imp: self, "unknown \'field\' value in caps, {caps:?}" ); return false; } } } else { Cea608Format::RawField0 } } Ok("s334-1a") => Cea608Format::S334_1A, v => { gst::error!( CAT, imp: self, "unknown or missing \'format\' value {v:?} in caps, {caps:?}" ); return false; } }; drop(state); let mut caps_builder = gst::Caps::builder("closedcaption/x-cea-708").field("format", "cc_data"); if let Some(framerate) = framerate { caps_builder = caps_builder.field("framerate", framerate); } let new_event = gst::event::Caps::new(&caps_builder.build()); return self.srcpad.push_event(new_event); } EventView::FlushStop(..) => { let mut state = self.state.borrow_mut(); let cea608_format = state.cea608.format; *state = State::default(); state.cea608.format = cea608_format; } _ => (), } gst::Pad::event_default(pad, Some(&*self.obj()), event) } } #[glib::object_subclass] impl ObjectSubclass for Cea608ToCea708 { const NAME: &'static str = "GstCea608ToCea708"; type Type = super::Cea608ToCea708; type ParentType = gst::Element; fn with_class(klass: &Self::Class) -> Self { let templ = klass.pad_template("sink").unwrap(); let sinkpad = gst::Pad::builder_with_template(&templ, Some("sink")) .chain_function(|pad, parent, buffer| { Cea608ToCea708::catch_panic_pad_function( parent, || Err(gst::FlowError::Error), |this| this.sink_chain(pad, buffer), ) }) .event_function(|pad, parent, event| { Cea608ToCea708::catch_panic_pad_function( parent, || false, |this| this.sink_event(pad, event), ) }) .flags(gst::PadFlags::FIXED_CAPS) .build(); let templ = klass.pad_template("src").unwrap(); let srcpad = gst::Pad::builder_with_template(&templ, Some("src")) .flags(gst::PadFlags::FIXED_CAPS) .build(); Self { srcpad, sinkpad, state: AtomicRefCell::new(State::default()), } } } impl ObjectImpl for Cea608ToCea708 { fn constructed(&self) { self.parent_constructed(); let obj = self.obj(); obj.add_pad(&self.sinkpad).unwrap(); obj.add_pad(&self.srcpad).unwrap(); } } impl GstObjectImpl for Cea608ToCea708 {} impl ElementImpl for Cea608ToCea708 { fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { static ELEMENT_METADATA: Lazy = Lazy::new(|| { gst::subclass::ElementMetadata::new( "CEA-608 to CEA-708", "Converter", "Converts CEA-608 Closed Captions to CEA-708 Closed Captions", "Matthew Waters ", ) }); Some(&*ELEMENT_METADATA) } fn pad_templates() -> &'static [gst::PadTemplate] { static PAD_TEMPLATES: Lazy> = Lazy::new(|| { let caps = gst::Caps::builder("closedcaption/x-cea-708") .field("format", "cc_data") .build(); let src_pad_template = gst::PadTemplate::new( "src", gst::PadDirection::Src, gst::PadPresence::Always, &caps, ) .unwrap(); let sink_pad_template = gst::PadTemplate::new( "sink", gst::PadDirection::Sink, gst::PadPresence::Always, &[ gst::Structure::builder("closedcaption/x-cea-608") .field("format", "s334-1a") .build(), gst::Structure::builder("closedcaption/x-cea-608") .field("format", "raw") .field("field", gst::List::new([0, 1])) .build(), ] .into_iter() .collect::(), ) .unwrap(); vec![src_pad_template, sink_pad_template] }); PAD_TEMPLATES.as_ref() } #[allow(clippy::single_match)] fn change_state( &self, transition: gst::StateChange, ) -> Result { gst::trace!(CAT, imp: self, "Changing state {:?}", transition); match transition { gst::StateChange::ReadyToPaused => { let mut state = self.state.borrow_mut(); *state = State::default(); } _ => (), } let ret = self.parent_change_state(transition)?; match transition { gst::StateChange::PausedToReady => { let mut state = self.state.borrow_mut(); *state = State::default(); } _ => (), } Ok(ret) } }