From 15e7a63e7bb00f837df3f251d28e9cba013b7e77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Tue, 9 Jan 2024 09:36:25 -0500 Subject: [PATCH] originalbuffer: Pair of elements to keep and restore original buffer The goal is to be able to get back the original buffer after performing analysis on a transformed version. Then put the various GstMeta back on the original buffer. An example pipeline would be .. ! originalbuffersave ! videoscale ! analysis ! originalbufferestore ! draw_overlay ! sink Part-of: --- Cargo.lock | 12 + Cargo.toml | 2 + docs/plugins/gst_plugins_cache.json | 62 ++++ generic/originalbuffer/Cargo.toml | 43 +++ generic/originalbuffer/build.rs | 3 + generic/originalbuffer/src/lib.rs | 38 +++ .../originalbuffer/src/originalbuffermeta.rs | 199 +++++++++++ .../src/originalbufferrestore/imp.rs | 315 ++++++++++++++++++ .../src/originalbufferrestore/mod.rs | 31 ++ .../src/originalbuffersave/imp.rs | 205 ++++++++++++ .../src/originalbuffersave/mod.rs | 41 +++ meson.build | 1 + meson_options.txt | 1 + 13 files changed, 953 insertions(+) create mode 100644 generic/originalbuffer/Cargo.toml create mode 100644 generic/originalbuffer/build.rs create mode 100644 generic/originalbuffer/src/lib.rs create mode 100644 generic/originalbuffer/src/originalbuffermeta.rs create mode 100644 generic/originalbuffer/src/originalbufferrestore/imp.rs create mode 100644 generic/originalbuffer/src/originalbufferrestore/mod.rs create mode 100644 generic/originalbuffer/src/originalbuffersave/imp.rs create mode 100644 generic/originalbuffer/src/originalbuffersave/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 97ec9c2c..94a50ff3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2556,6 +2556,18 @@ dependencies = [ "xmltree", ] +[[package]] +name = "gst-plugin-originalbuffer" +version = "0.13.0-alpha.1" +dependencies = [ + "atomic_refcell", + "glib", + "gst-plugin-version-helper", + "gstreamer", + "gstreamer-video", + "once_cell", +] + [[package]] name = "gst-plugin-png" version = "0.13.0-alpha.1" diff --git a/Cargo.toml b/Cargo.toml index 574007e5..cc0d62bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "audio/spotify", "generic/file", + "generic/originalbuffer", "generic/sodium", "generic/threadshare", "generic/inter", @@ -65,6 +66,7 @@ default-members = [ "audio/claxon", "audio/lewton", + "generic/originalbuffer", "generic/threadshare", "generic/inter", diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index c9989153..c4dacb05 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -3709,6 +3709,68 @@ "tracers": {}, "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" }, + "originalbuffer": { + "description": "GStreamer Origin buffer meta Plugin", + "elements": { + "originalbufferrestore": { + "author": "Olivier Crête ", + "description": "Restores a reference to the buffer in a meta", + "hierarchy": [ + "GstOriginalBufferRestore", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Generic", + "pad-templates": { + "sink": { + "caps": "ANY", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "ANY", + "direction": "src", + "presence": "always" + } + }, + "rank": "none" + }, + "originalbuffersave": { + "author": "Olivier Crête ", + "description": "Saves a reference to the buffer in a meta", + "hierarchy": [ + "GstOriginalBufferSave", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Generic", + "pad-templates": { + "sink": { + "caps": "ANY", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "ANY", + "direction": "src", + "presence": "always" + } + }, + "rank": "none" + } + }, + "filename": "gstoriginalbuffer", + "license": "MPL", + "other-types": {}, + "package": "gst-plugin-originalbuffer", + "source": "gst-plugin-originalbuffer", + "tracers": {}, + "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" + }, "raptorq": { "description": "GStreamer RaptorQ FEC Plugin", "elements": { diff --git a/generic/originalbuffer/Cargo.toml b/generic/originalbuffer/Cargo.toml new file mode 100644 index 00000000..d1a7b4e1 --- /dev/null +++ b/generic/originalbuffer/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "gst-plugin-originalbuffer" +version.workspace = true +authors = ["Olivier Crête "] +repository.workspace = true +license = "MPL-2.0" +description = "GStreamer Origin buffer meta Plugin" +edition.workspace = true +rust-version.workspace = true + +[dependencies] +glib.workspace = true +gst.workspace = true +gst-video.workspace = true +atomic_refcell = "0.1" +once_cell.workspace = true + +[lib] +name = "gstoriginalbuffer" +crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" + +[build-dependencies] +gst-plugin-version-helper.workspace = true + +[features] +static = [] +capi = [] +doc = ["gst/v1_16"] + +[package.metadata.capi] +min_version = "0.9.21" + +[package.metadata.capi.header] +enabled = false + +[package.metadata.capi.library] +install_subdir = "gstreamer-1.0" +versioning = false +import_library = false + +[package.metadata.capi.pkg_config] +requires_private = "gstreamer-1.0, gstreamer-base-1.0, gobject-2.0, glib-2.0, gmodule-2.0" diff --git a/generic/originalbuffer/build.rs b/generic/originalbuffer/build.rs new file mode 100644 index 00000000..cda12e57 --- /dev/null +++ b/generic/originalbuffer/build.rs @@ -0,0 +1,3 @@ +fn main() { + gst_plugin_version_helper::info() +} diff --git a/generic/originalbuffer/src/lib.rs b/generic/originalbuffer/src/lib.rs new file mode 100644 index 00000000..428083be --- /dev/null +++ b/generic/originalbuffer/src/lib.rs @@ -0,0 +1,38 @@ +// Copyright (C) 2024 Collabora Ltd +// @author: Olivier Crête +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +#![allow(clippy::non_send_fields_in_send_ty, unused_doc_comments)] + +/** + * plugin-originalbuffer: + * + * Since: plugins-rs-0.12 */ +use gst::glib; + +mod originalbuffermeta; +mod originalbufferrestore; +mod originalbuffersave; + +fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + originalbuffersave::register(plugin)?; + originalbufferrestore::register(plugin)?; + Ok(()) +} + +gst::plugin_define!( + originalbuffer, + env!("CARGO_PKG_DESCRIPTION"), + plugin_init, + concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")), + "MPL", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_REPOSITORY"), + env!("BUILD_REL_DATE") +); diff --git a/generic/originalbuffer/src/originalbuffermeta.rs b/generic/originalbuffer/src/originalbuffermeta.rs new file mode 100644 index 00000000..c94adcda --- /dev/null +++ b/generic/originalbuffer/src/originalbuffermeta.rs @@ -0,0 +1,199 @@ +// Copyright (C) 2024 Collabora Ltd +// @author: Olivier Crête +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::prelude::*; +use std::fmt; +use std::mem; + +#[repr(transparent)] +pub struct OriginalBufferMeta(imp::OriginalBufferMeta); + +unsafe impl Send for OriginalBufferMeta {} +unsafe impl Sync for OriginalBufferMeta {} + +impl OriginalBufferMeta { + pub fn add( + buffer: &mut gst::BufferRef, + original: gst::Buffer, + caps: Option, + ) -> gst::MetaRefMut<'_, Self, gst::meta::Standalone> { + unsafe { + // Manually dropping because gst_buffer_add_meta() takes ownership of the + // content of the struct + let mut params = + mem::ManuallyDrop::new(imp::OriginalBufferMetaParams { original, caps }); + + let meta = gst::ffi::gst_buffer_add_meta( + buffer.as_mut_ptr(), + imp::original_buffer_meta_get_info(), + &mut *params as *mut imp::OriginalBufferMetaParams as gst::glib::ffi::gpointer, + ) as *mut imp::OriginalBufferMeta; + + Self::from_mut_ptr(buffer, meta) + } + } + + pub fn replace(&mut self, original: gst::Buffer, caps: Option) { + self.0.original = Some(original); + self.0.caps = caps; + } + + pub fn original(&self) -> &gst::Buffer { + self.0.original.as_ref().unwrap() + } + + pub fn caps(&self) -> &gst::Caps { + self.0.caps.as_ref().unwrap() + } +} + +unsafe impl MetaAPI for OriginalBufferMeta { + type GstType = imp::OriginalBufferMeta; + + fn meta_api() -> gst::glib::Type { + imp::original_buffer_meta_api_get_type() + } +} + +impl fmt::Debug for OriginalBufferMeta { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("OriginalBufferMeta") + .field("buffer", &self.original()) + .finish() + } +} + +mod imp { + use gst::glib::translate::*; + use once_cell::sync::Lazy; + use std::mem; + use std::ptr; + + pub(super) struct OriginalBufferMetaParams { + pub original: gst::Buffer, + pub caps: Option, + } + + #[repr(C)] + pub struct OriginalBufferMeta { + parent: gst::ffi::GstMeta, + pub(super) original: Option, + pub(super) caps: Option, + } + + pub(super) fn original_buffer_meta_api_get_type() -> glib::Type { + static TYPE: Lazy = Lazy::new(|| unsafe { + let t = from_glib(gst::ffi::gst_meta_api_type_register( + b"GstOriginalBufferMetaAPI\0".as_ptr() as *const _, + [ptr::null::()].as_ptr() as *mut *const _, + )); + + assert_ne!(t, glib::Type::INVALID); + + t + }); + + *TYPE + } + + unsafe extern "C" fn original_buffer_meta_init( + meta: *mut gst::ffi::GstMeta, + params: glib::ffi::gpointer, + _buffer: *mut gst::ffi::GstBuffer, + ) -> glib::ffi::gboolean { + assert!(!params.is_null()); + let meta = &mut *(meta as *mut OriginalBufferMeta); + let params = ptr::read(params as *const OriginalBufferMetaParams); + + let OriginalBufferMetaParams { original, caps } = params; + + ptr::write(&mut meta.original, Some(original)); + ptr::write(&mut meta.caps, caps); + + true.into_glib() + } + + unsafe extern "C" fn original_buffer_meta_free( + meta: *mut gst::ffi::GstMeta, + _buffer: *mut gst::ffi::GstBuffer, + ) { + let meta = &mut *(meta as *mut OriginalBufferMeta); + meta.original = None; + meta.caps = None; + } + + unsafe extern "C" fn original_buffer_meta_transform( + dest: *mut gst::ffi::GstBuffer, + meta: *mut gst::ffi::GstMeta, + _buffer: *mut gst::ffi::GstBuffer, + _type_: glib::ffi::GQuark, + _data: glib::ffi::gpointer, + ) -> glib::ffi::gboolean { + let dest = gst::BufferRef::from_mut_ptr(dest); + let meta = &*(meta as *const OriginalBufferMeta); + + if dest.meta::().is_some() { + return true.into_glib(); + } + // We don't store a ref in the meta if it's self-refencing, but we add it + // when copying the meta to another buffer. + super::OriginalBufferMeta::add( + dest, + meta.original.as_ref().unwrap().clone(), + meta.caps.clone(), + ); + + true.into_glib() + } + + pub(super) fn original_buffer_meta_get_info() -> *const gst::ffi::GstMetaInfo { + struct MetaInfo(ptr::NonNull); + unsafe impl Send for MetaInfo {} + unsafe impl Sync for MetaInfo {} + + static META_INFO: Lazy = Lazy::new(|| unsafe { + MetaInfo( + ptr::NonNull::new(gst::ffi::gst_meta_register( + original_buffer_meta_api_get_type().into_glib(), + b"OriginalBufferMeta\0".as_ptr() as *const _, + mem::size_of::(), + Some(original_buffer_meta_init), + Some(original_buffer_meta_free), + Some(original_buffer_meta_transform), + ) as *mut gst::ffi::GstMetaInfo) + .expect("Failed to register meta API"), + ) + }); + + META_INFO.0.as_ptr() + } +} + +#[test] +fn test() { + gst::init().unwrap(); + let mut b = gst::Buffer::with_size(10).unwrap(); + let caps = gst::Caps::new_empty_simple("video/x-raw"); + let copy = b.copy(); + let m = OriginalBufferMeta::add(b.make_mut(), copy, Some(caps.clone())); + assert_eq!(m.caps(), caps.as_ref()); + assert_eq!(m.original().clone(), b); + let b2: gst::Buffer = b.copy_deep().unwrap(); + let m = b.meta::().unwrap(); + assert_eq!(m.caps(), caps.as_ref()); + assert_eq!(m.original(), &b); + let m = b2.meta::().unwrap(); + assert_eq!(m.caps(), caps.as_ref()); + assert_eq!(m.original(), &b); + let b3: gst::Buffer = b2.copy_deep().unwrap(); + drop(b2); + let m = b3.meta::().unwrap(); + assert_eq!(m.caps(), caps.as_ref()); + assert_eq!(m.original(), &b); +} diff --git a/generic/originalbuffer/src/originalbufferrestore/imp.rs b/generic/originalbuffer/src/originalbufferrestore/imp.rs new file mode 100644 index 00000000..8924ff4b --- /dev/null +++ b/generic/originalbuffer/src/originalbufferrestore/imp.rs @@ -0,0 +1,315 @@ +// Copyright (C) 2024 Collabora Ltd +// @author: Olivier Crête +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::subclass::prelude::*; +use gst_video::prelude::*; + +use atomic_refcell::AtomicRefCell; + +use crate::originalbuffermeta; +use crate::originalbuffermeta::OriginalBufferMeta; + +struct CapsState { + caps: gst::Caps, + vinfo: Option, +} + +impl Default for CapsState { + fn default() -> Self { + CapsState { + caps: gst::Caps::new_empty(), + vinfo: None, + } + } +} + +#[derive(Default)] +struct State { + sinkpad_caps: CapsState, + meta_caps: CapsState, + sinkpad_segment: Option, +} + +pub struct OriginalBufferRestore { + state: AtomicRefCell, + src_pad: gst::Pad, + sink_pad: gst::Pad, +} + +use once_cell::sync::Lazy; +#[cfg(unused_code)] +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "originalbufferrestore", + gst::DebugColorFlags::empty(), + Some("Restore Original buffer as meta"), + ) +}); + +#[glib::object_subclass] +impl ObjectSubclass for OriginalBufferRestore { + const NAME: &'static str = "GstOriginalBufferRestore"; + type Type = super::OriginalBufferRestore; + type ParentType = gst::Element; + + fn with_class(klass: &Self::Class) -> Self { + let sink_templ = klass.pad_template("sink").unwrap(); + let src_templ = klass.pad_template("src").unwrap(); + + let sink_pad = gst::Pad::builder_from_template(&sink_templ) + .chain_function(|pad, parent, buffer| { + OriginalBufferRestore::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |obj| obj.sink_chain(pad, buffer), + ) + }) + .event_function(|pad, parent, event| { + OriginalBufferRestore::catch_panic_pad_function( + parent, + || false, + |obj| obj.sink_event(pad, parent, event), + ) + }) + .query_function(|pad, parent, query| { + OriginalBufferRestore::catch_panic_pad_function( + parent, + || false, + |obj| obj.sink_query(pad, parent, query), + ) + }) + .build(); + + let src_pad = gst::Pad::builder_from_template(&src_templ) + .event_function(|pad, parent, event| { + OriginalBufferRestore::catch_panic_pad_function( + parent, + || false, + |obj| obj.src_event(pad, parent, event), + ) + }) + .build(); + + Self { + src_pad, + sink_pad, + state: Default::default(), + } + } +} + +impl ObjectImpl for OriginalBufferRestore { + fn constructed(&self) { + self.parent_constructed(); + + let obj = self.obj(); + obj.add_pad(&self.sink_pad).unwrap(); + obj.add_pad(&self.src_pad).unwrap(); + } +} + +impl GstObjectImpl for OriginalBufferRestore {} + +impl ElementImpl for OriginalBufferRestore { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "Original Buffer Restore", + "Generic", + "Restores a reference to the buffer in a meta", + "Olivier Crête ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let caps = gst::Caps::new_any(); + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } + + fn change_state( + &self, + transition: gst::StateChange, + ) -> Result { + let ret = self.parent_change_state(transition)?; + if transition == gst::StateChange::PausedToReady { + let mut state = self.state.borrow_mut(); + *state = State::default(); + } + + Ok(ret) + } +} + +impl OriginalBufferRestore { + fn sink_event( + &self, + pad: &gst::Pad, + parent: Option<&impl IsA>, + event: gst::Event, + ) -> bool { + match event.view() { + gst::EventView::Caps(e) => { + let mut state = self.state.borrow_mut(); + + let caps = e.caps_owned(); + let vinfo = gst_video::VideoInfo::from_caps(&caps).ok(); + state.sinkpad_caps = CapsState { caps, vinfo }; + true + } + gst::EventView::Segment(_) => { + let mut state = self.state.borrow_mut(); + state.sinkpad_segment = Some(event); + true + } + _ => gst::Pad::event_default(pad, parent, event), + } + } + + fn src_event( + &self, + pad: &gst::Pad, + parent: Option<&impl IsA>, + event: gst::Event, + ) -> bool { + if event.type_() == gst::EventType::Reconfigure + || event.has_name("gst-original-buffer-forward-upstream-event") + { + let s = gst::Structure::builder("gst-original-buffer-forward-upstream-event") + .field("event", event) + .build(); + let event = gst::event::CustomUpstream::new(s); + self.sink_pad.push_event(event) + } else { + gst::Pad::event_default(pad, parent, event) + } + } + + fn sink_query( + &self, + pad: &gst::Pad, + parent: Option<&impl IsA>, + query: &mut gst::QueryRef, + ) -> bool { + if let gst::QueryViewMut::Custom(_) = query.view_mut() { + let s = query.structure_mut(); + if s.has_name("gst-original-buffer-forward-query") { + if let Ok(mut q) = s.get::("query") { + s.remove_field("query"); + assert!(q.is_writable()); + let res = self.src_pad.peer_query(q.get_mut().unwrap()); + + s.set("query", q); + s.set("result", res); + + return true; + } + } + } + + gst::Pad::query_default(pad, parent, query) + } + + fn sink_chain( + &self, + _pad: &gst::Pad, + inbuf: gst::Buffer, + ) -> Result { + let Some(ometa) = inbuf.meta::() else { + //gst::element_warning!(self, gst::StreamError::Failed, ["Buffer {} is missing the GstOriginalBufferMeta, put originalbuffersave upstream in your pipeline", buffer]); + return Ok(gst::FlowSuccess::Ok); + }; + let mut state = self.state.borrow_mut(); + let meta_caps = &mut state.meta_caps; + if &meta_caps.caps != ometa.caps() { + if !self.src_pad.push_event(gst::event::Caps::new(ometa.caps())) { + return Err(gst::FlowError::NotNegotiated); + } + meta_caps.caps = ometa.caps().clone(); + meta_caps.vinfo = gst_video::VideoInfo::from_caps(&meta_caps.caps).ok(); + } + + let mut outbuf = ometa.original().copy(); + + inbuf + .copy_into( + outbuf.make_mut(), + gst::BufferCopyFlags::TIMESTAMPS | gst::BufferCopyFlags::FLAGS, + .., + ) + .unwrap(); + + for meta in inbuf.iter_meta::() { + if meta.api() == originalbuffermeta::OriginalBufferMeta::meta_api() { + continue; + } + + if meta.has_tag::() + || meta.has_tag::() + { + continue; + } + + if meta.has_tag::() { + if let (Some(ref meta_vinfo), Some(ref sink_vinfo)) = + (&state.meta_caps.vinfo, &state.sinkpad_caps.vinfo) + { + if (meta_vinfo.width() != sink_vinfo.width() + || meta_vinfo.height() != sink_vinfo.height()) + && meta + .transform( + outbuf.make_mut(), + &gst_video::video_meta::VideoMetaTransform::new( + sink_vinfo, meta_vinfo, + ), + ) + .is_ok() + { + continue; + } + } + } + + let _ = meta.transform( + outbuf.make_mut(), + &gst::meta::MetaTransformCopy::new(false, ..), + ); + } + + if let Some(event) = state.sinkpad_segment.take() { + if !self.src_pad.push_event(event) { + return Err(gst::FlowError::Error); + } + } + + self.src_pad.push(outbuf) + } +} diff --git a/generic/originalbuffer/src/originalbufferrestore/mod.rs b/generic/originalbuffer/src/originalbufferrestore/mod.rs new file mode 100644 index 00000000..59666b7d --- /dev/null +++ b/generic/originalbuffer/src/originalbufferrestore/mod.rs @@ -0,0 +1,31 @@ +// Copyright (C) 2024 Collabora Ltd +// @author: Olivier Crête +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +/** + * SECTION:element-originalbufferrestore + * + * See originalbuffersave for details + */ +use gst::glib; +use gst::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct OriginalBufferRestore(ObjectSubclass) @extends gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "originalbufferrestore", + gst::Rank::NONE, + OriginalBufferRestore::static_type(), + ) +} diff --git a/generic/originalbuffer/src/originalbuffersave/imp.rs b/generic/originalbuffer/src/originalbuffersave/imp.rs new file mode 100644 index 00000000..6d77614d --- /dev/null +++ b/generic/originalbuffer/src/originalbuffersave/imp.rs @@ -0,0 +1,205 @@ +// Copyright (C) 2024 Collabora Ltd +// @author: Olivier Crête +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; +use gst::subclass::prelude::*; + +use crate::originalbuffermeta::OriginalBufferMeta; + +pub struct OriginalBufferSave { + src_pad: gst::Pad, + sink_pad: gst::Pad, +} + +use once_cell::sync::Lazy; +#[cfg(unused_code)] +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "originalbuffersave", + gst::DebugColorFlags::empty(), + Some("Save Original buffer as meta"), + ) +}); + +#[glib::object_subclass] +impl ObjectSubclass for OriginalBufferSave { + const NAME: &'static str = "GstOriginalBufferSave"; + type Type = super::OriginalBufferSave; + type ParentType = gst::Element; + + fn with_class(klass: &Self::Class) -> Self { + let sink_templ = klass.pad_template("sink").unwrap(); + let src_templ = klass.pad_template("src").unwrap(); + + let sink_pad = gst::Pad::builder_from_template(&sink_templ) + .chain_function(|pad, parent, buffer| { + OriginalBufferSave::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |obj| obj.sink_chain(pad, buffer), + ) + }) + .query_function(|pad, parent, query| { + OriginalBufferSave::catch_panic_pad_function( + parent, + || false, + |obj| obj.sink_query(pad, parent, query), + ) + }) + .flags(gst::PadFlags::PROXY_CAPS | gst::PadFlags::PROXY_ALLOCATION) + .build(); + + let src_pad = gst::Pad::builder_from_template(&src_templ) + .event_function(|pad, parent, event| { + OriginalBufferSave::catch_panic_pad_function( + parent, + || false, + |obj| obj.src_event(pad, parent, event), + ) + }) + .build(); + + Self { src_pad, sink_pad } + } +} + +impl ObjectImpl for OriginalBufferSave { + fn constructed(&self) { + self.parent_constructed(); + + let obj = self.obj(); + obj.add_pad(&self.sink_pad).unwrap(); + obj.add_pad(&self.src_pad).unwrap(); + } +} + +impl GstObjectImpl for OriginalBufferSave {} + +impl ElementImpl for OriginalBufferSave { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "Original Buffer Save", + "Generic", + "Saves a reference to the buffer in a meta", + "Olivier Crête ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let caps = gst::Caps::new_any(); + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl OriginalBufferSave { + fn forward_query(&self, query: gst::Query) -> Option { + let mut s = gst::Structure::new_empty("gst-original-buffer-forward-query"); + s.set("query", query); + + let mut query = gst::query::Custom::new(s); + if self.src_pad.peer_query(&mut query) { + let s = query.structure_mut(); + if let (Ok(true), Ok(q)) = (s.get("result"), s.get::("query")) { + Some(q) + } else { + None + } + } else { + None + } + } + + fn sink_chain( + &self, + pad: &gst::Pad, + inbuf: gst::Buffer, + ) -> Result { + let mut buf = inbuf.copy(); + let caps = pad.current_caps(); + + if let Some(mut meta) = buf.make_mut().meta_mut::() { + meta.replace(inbuf, caps); + } else { + OriginalBufferMeta::add(buf.make_mut(), inbuf, caps); + } + + self.src_pad.push(buf) + } + + fn sink_query( + &self, + pad: &gst::Pad, + parent: Option<&impl IsA>, + query: &mut gst::QueryRef, + ) -> bool { + let ret = gst::Pad::query_default(pad, parent, query); + if !ret { + return ret; + } + + if let gst::QueryViewMut::Caps(q) = query.view_mut() { + if let Some(caps) = q.result_owned() { + let forwarding_q = gst::query::Caps::new(Some(&caps)).into(); + + if let Some(forwarding_q) = self.forward_query(forwarding_q) { + if let gst::QueryView::Caps(c) = forwarding_q.view() { + let res = c + .result_owned() + .map(|c| c.intersect_with_mode(&caps, gst::CapsIntersectMode::First)); + q.set_result(&res); + } + } + } + } + + // We should also do allocation queries, but that requires supporting the same + // intersection semantics as gsttee, which should be in a helper function. + + true + } + + fn src_event( + &self, + pad: &gst::Pad, + parent: Option<&impl IsA>, + event: gst::Event, + ) -> bool { + let event = if event.has_name("gst-original-buffer-forward-upstream-event") { + event.structure().unwrap().get("event").unwrap() + } else { + event + }; + + gst::Pad::event_default(pad, parent, event) + } +} diff --git a/generic/originalbuffer/src/originalbuffersave/mod.rs b/generic/originalbuffer/src/originalbuffersave/mod.rs new file mode 100644 index 00000000..004c45b7 --- /dev/null +++ b/generic/originalbuffer/src/originalbuffersave/mod.rs @@ -0,0 +1,41 @@ +// Copyright (C) 2024 Collabora Ltd +// @author: Olivier Crête +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +/** + * SECTION:element-originalbuffersave + * + * GStreamer elements to store the original buffer and restore it later + * + * In many analysis scenario (for example machine learning), it is desirable to + * use a pre-processed buffer, for example by lowering the resolution, but we may + * want to take the output of this analysis, and apply it to the original buffer. + * + * These elements do just this, the typical usage would be a pipeline like: + * + * `... ! originalbuffersave ! videoconvertscale ! video/x-raw, width=100, height=100 ! analysiselement ! originalbufferrestore ! ...` + * + * The originalbufferrestore element will "restore" the buffer that was entered to the "save" element, but will keep any metadata that was added later. + */ +use gst::glib; +use gst::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct OriginalBufferSave(ObjectSubclass) @extends gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "originalbuffersave", + gst::Rank::NONE, + OriginalBufferSave::static_type(), + ) +} diff --git a/meson.build b/meson.build index 7a256f85..e6b7a366 100644 --- a/meson.build +++ b/meson.build @@ -118,6 +118,7 @@ plugins = { 'spotify': {'library': 'libgstspotify'}, 'file': {'library': 'libgstrsfile'}, + 'originalbuffer': {'library': 'libgstoriginalbuffer'}, # sodium can have an external dependency, see below 'threadshare': { 'library': 'libgstthreadshare', diff --git a/meson_options.txt b/meson_options.txt index 0842a661..94e4ae01 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -9,6 +9,7 @@ option('spotify', type: 'feature', value: 'auto', description: 'Build spotify pl # generic option('file', type: 'feature', value: 'auto', description: 'Build file plugin') +option('originalbuffer', type: 'feature', value: 'auto', description: 'Build originalbuffer plugin') option('sodium', type: 'feature', value: 'auto', description: 'Build sodium plugin') option('sodium-source', type: 'combo', choices: ['system', 'built-in'], value: 'built-in',