diff --git a/Cargo.toml b/Cargo.toml index 4f296fe7..eb62ea94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "video/gif", "video/rav1e", "video/rspng", + "video/hsv", "text/wrap", ] diff --git a/meson.build b/meson.build index b0f1781c..36f104a3 100644 --- a/meson.build +++ b/meson.build @@ -42,6 +42,7 @@ plugins_rep = { 'gst-plugin-textwrap': 'libgstrstextwrap', 'gst-plugin-threadshare': 'libgstthreadshare', 'gst-plugin-togglerecord': 'libgsttogglerecord', + 'gst-plugin-hsv': 'libgsthsv', } exclude = [] diff --git a/video/hsv/Cargo.toml b/video/hsv/Cargo.toml new file mode 100644 index 00000000..bdcf6b9d --- /dev/null +++ b/video/hsv/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "gst-plugin-hsv" +version = "0.6.0" +authors = ["Julien Bardagi "] +repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs" +license = "MIT/Apache-2.0" +edition = "2018" +description = "HSV manipulation elements, written in Rust" + +[dependencies] +glib = { git = "https://github.com/gtk-rs/gtk-rs" } +gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst-video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +byte-slice-cast = "1.0" +atomic_refcell = "0.1" +num-traits = "0.2" +once_cell = "1.0" + +[dev-dependencies] +gst_check = { package = "gstreamer-check", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } + +[lib] +name = "gsthsv" +crate-type = ["cdylib", "rlib", "staticlib"] +path = "src/lib.rs" + +[build-dependencies] +gst-plugin-version-helper = { path="../../version-helper" } diff --git a/video/hsv/LICENSE-APACHE b/video/hsv/LICENSE-APACHE new file mode 100644 index 00000000..16fe87b0 --- /dev/null +++ b/video/hsv/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/video/hsv/LICENSE-MIT b/video/hsv/LICENSE-MIT new file mode 100644 index 00000000..31aa7938 --- /dev/null +++ b/video/hsv/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/video/hsv/build.rs b/video/hsv/build.rs new file mode 100644 index 00000000..17be1215 --- /dev/null +++ b/video/hsv/build.rs @@ -0,0 +1,3 @@ +fn main() { + gst_plugin_version_helper::get_info() +} diff --git a/video/hsv/src/hsvdetector/imp.rs b/video/hsv/src/hsvdetector/imp.rs new file mode 100644 index 00000000..a82400fa --- /dev/null +++ b/video/hsv/src/hsvdetector/imp.rs @@ -0,0 +1,537 @@ +// Copyright (C) 2020 Julien Bardagi +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use glib::subclass; +use glib::subclass::prelude::*; +use gst::prelude::*; +use gst::subclass::prelude::*; + +use gst::{gst_debug, gst_info}; +use gst_base::subclass::prelude::*; + +use atomic_refcell::AtomicRefCell; +use std::i32; +use std::sync::Mutex; + +use once_cell::sync::Lazy; +use std::convert::TryInto; + +use super::super::hsvutils; + +// Default values of properties +const DEFAULT_HUE_REF: f32 = 0.0; +const DEFAULT_HUE_VAR: f32 = 10.0; +const DEFAULT_SATURATION_REF: f32 = 0.0; +const DEFAULT_SATURATION_VAR: f32 = 0.15; +const DEFAULT_VALUE_REF: f32 = 0.0; +const DEFAULT_VALUE_VAR: f32 = 0.3; + +// Property value storage +#[derive(Debug, Clone, Copy)] +struct Settings { + hue_ref: f32, + hue_var: f32, + saturation_ref: f32, + saturation_var: f32, + value_ref: f32, + value_var: f32, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + hue_ref: DEFAULT_HUE_REF, + hue_var: DEFAULT_HUE_VAR, + saturation_ref: DEFAULT_SATURATION_REF, + saturation_var: DEFAULT_SATURATION_VAR, + value_ref: DEFAULT_VALUE_REF, + value_var: DEFAULT_VALUE_VAR, + } + } +} + +// Metadata for the properties +static PROPERTIES: [subclass::Property; 6] = [ + subclass::Property("hue-ref", |name| { + glib::ParamSpec::float( + name, + "Hue reference", + "Hue reference in degrees", + f32::MIN, + f32::MAX, + DEFAULT_HUE_REF, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("hue-var", |name| { + glib::ParamSpec::float( + name, + "Hue variation", + "Allowed hue variation from the reference hue angle, in degrees", + 0.0, + 180.0, + DEFAULT_HUE_VAR, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("saturation-ref", |name| { + glib::ParamSpec::float( + name, + "Saturation reference", + "Reference saturation value", + 0.0, + 1.0, + DEFAULT_SATURATION_REF, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("saturation-var", |name| { + glib::ParamSpec::float( + name, + "Saturation variation", + "Allowed saturation variation from the reference value", + 0.0, + 1.0, + DEFAULT_SATURATION_VAR, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("value-ref", |name| { + glib::ParamSpec::float( + name, + "Value reference", + "Reference value value", + 0.0, + 1.0, + DEFAULT_VALUE_REF, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("value-var", |name| { + glib::ParamSpec::float( + name, + "Value variation", + "Allowed value variation from the reference value", + 0.0, + 1.0, + DEFAULT_VALUE_VAR, + glib::ParamFlags::READWRITE, + ) + }), +]; + +// Stream-specific state, i.e. video format configuration +struct State { + in_info: gst_video::VideoInfo, + out_info: gst_video::VideoInfo, +} + +// Struct containing all the element data +pub struct HsvDetector { + settings: Mutex, + state: AtomicRefCell>, +} + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "hsvdetector", + gst::DebugColorFlags::empty(), + Some("Rust HSV-based detection filter"), + ) +}); + +impl ObjectSubclass for HsvDetector { + const NAME: &'static str = "HsvDetector"; + type Type = super::HsvDetector; + type ParentType = gst_base::BaseTransform; + type Instance = gst::subclass::ElementInstanceStruct; + type Class = subclass::simple::ClassStruct; + + // Boilerplate macro + glib::object_subclass!(); + + // Creates a new instance. + fn new() -> Self { + Self { + settings: Mutex::new(Default::default()), + state: AtomicRefCell::new(None), + } + } + + fn class_init(klass: &mut Self::Class) { + klass.set_metadata( + "HSV detector", + "Filter/Effect/Converter/Video", + "Works within the HSV colorspace to mark positive pixels", + "Julien Bardagi ", + ); + + // src pad capabilities + let caps = gst::Caps::new_simple( + "video/x-raw", + &[ + ( + "format", + &gst::List::new(&[&gst_video::VideoFormat::Rgba.to_str()]), + ), + ("width", &gst::IntRange::::new(0, i32::MAX)), + ("height", &gst::IntRange::::new(0, i32::MAX)), + ( + "framerate", + &gst::FractionRange::new( + gst::Fraction::new(0, 1), + gst::Fraction::new(i32::MAX, 1), + ), + ), + ], + ); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + klass.add_pad_template(src_pad_template); + + // sink pad capabilities + let caps = gst::Caps::new_simple( + "video/x-raw", + &[ + ( + "format", + &gst::List::new(&[&gst_video::VideoFormat::Rgbx.to_str()]), + ), + ("width", &gst::IntRange::::new(0, i32::MAX)), + ("height", &gst::IntRange::::new(0, i32::MAX)), + ( + "framerate", + &gst::FractionRange::new( + gst::Fraction::new(0, 1), + gst::Fraction::new(i32::MAX, 1), + ), + ), + ], + ); + + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + klass.add_pad_template(sink_pad_template); + + // Install all our properties + klass.install_properties(&PROPERTIES); + + klass.configure( + gst_base::subclass::BaseTransformMode::NeverInPlace, + false, + false, + ); + } +} + +impl ObjectImpl for HsvDetector { + fn set_property(&self, obj: &Self::Type, id: usize, value: &glib::Value) { + let prop = &PROPERTIES[id]; + + match *prop { + subclass::Property("hue-ref", ..) => { + let mut settings = self.settings.lock().unwrap(); + let hue_ref = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing hue-ref from {} to {}", + settings.hue_ref, + hue_ref + ); + settings.hue_ref = hue_ref; + } + subclass::Property("hue-var", ..) => { + let mut settings = self.settings.lock().unwrap(); + let hue_var = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing hue-var from {} to {}", + settings.hue_var, + hue_var + ); + settings.hue_var = hue_var; + } + subclass::Property("saturation-ref", ..) => { + let mut settings = self.settings.lock().unwrap(); + let saturation_ref = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing saturation-ref from {} to {}", + settings.saturation_ref, + saturation_ref + ); + settings.saturation_ref = saturation_ref; + } + subclass::Property("saturation-var", ..) => { + let mut settings = self.settings.lock().unwrap(); + let saturation_var = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing saturation-var from {} to {}", + settings.saturation_var, + saturation_var + ); + settings.saturation_var = saturation_var; + } + subclass::Property("value-ref", ..) => { + let mut settings = self.settings.lock().unwrap(); + let value_ref = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing value-ref from {} to {}", + settings.value_ref, + value_ref + ); + settings.value_ref = value_ref; + } + subclass::Property("value-var", ..) => { + let mut settings = self.settings.lock().unwrap(); + let value_var = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing value-var from {} to {}", + settings.value_var, + value_var + ); + settings.value_var = value_var; + } + _ => unimplemented!(), + } + } + + // Called whenever a value of a property is read. It can be called + // at any time from any thread. + fn get_property(&self, _obj: &Self::Type, id: usize) -> glib::Value { + let prop = &PROPERTIES[id]; + + match *prop { + subclass::Property("hue-ref", ..) => { + let settings = self.settings.lock().unwrap(); + settings.hue_ref.to_value() + } + subclass::Property("hue-var", ..) => { + let settings = self.settings.lock().unwrap(); + settings.hue_var.to_value() + } + subclass::Property("saturation-ref", ..) => { + let settings = self.settings.lock().unwrap(); + settings.saturation_ref.to_value() + } + subclass::Property("saturation-var", ..) => { + let settings = self.settings.lock().unwrap(); + settings.saturation_var.to_value() + } + subclass::Property("value-ref", ..) => { + let settings = self.settings.lock().unwrap(); + settings.value_ref.to_value() + } + subclass::Property("value-var", ..) => { + let settings = self.settings.lock().unwrap(); + settings.value_var.to_value() + } + _ => unimplemented!(), + } + } +} + +impl ElementImpl for HsvDetector {} + +impl BaseTransformImpl for HsvDetector { + fn transform_caps( + &self, + element: &Self::Type, + direction: gst::PadDirection, + caps: &gst::Caps, + filter: Option<&gst::Caps>, + ) -> Option { + let other_caps = if direction == gst::PadDirection::Src { + let mut caps = caps.clone(); + + for s in caps.make_mut().iter_mut() { + s.set("format", &gst_video::VideoFormat::Rgbx.to_str()); + } + + caps + } else { + let mut caps = caps.clone(); + + for s in caps.make_mut().iter_mut() { + s.set("format", &gst_video::VideoFormat::Rgba.to_str()); + } + + caps + }; + + gst_debug!( + CAT, + obj: element, + "Transformed caps from {} to {} in direction {:?}", + caps, + other_caps, + direction + ); + + // In the end we need to filter the caps through an optional filter caps to get rid of any + // unwanted caps. + if let Some(filter) = filter { + Some(filter.intersect_with_mode(&other_caps, gst::CapsIntersectMode::First)) + } else { + Some(other_caps) + } + } + + fn get_unit_size(&self, _element: &Self::Type, caps: &gst::Caps) -> Option { + gst_video::VideoInfo::from_caps(caps) + .map(|info| info.size()) + .ok() + } + + fn set_caps( + &self, + element: &Self::Type, + incaps: &gst::Caps, + outcaps: &gst::Caps, + ) -> Result<(), gst::LoggableError> { + let in_info = match gst_video::VideoInfo::from_caps(incaps) { + Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse input caps")), + Ok(info) => info, + }; + let out_info = match gst_video::VideoInfo::from_caps(outcaps) { + Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse output caps")), + Ok(info) => info, + }; + + gst_debug!( + CAT, + obj: element, + "Configured for caps {} to {}", + incaps, + outcaps + ); + + *self.state.borrow_mut() = Some(State { in_info, out_info }); + + Ok(()) + } + + fn stop(&self, element: &Self::Type) -> Result<(), gst::ErrorMessage> { + // Drop state + *self.state.borrow_mut() = None; + + gst_info!(CAT, obj: element, "Stopped"); + + Ok(()) + } + + fn transform( + &self, + element: &Self::Type, + inbuf: &gst::Buffer, + outbuf: &mut gst::BufferRef, + ) -> Result { + let settings = *self.settings.lock().unwrap(); + + let mut state_guard = self.state.borrow_mut(); + let state = state_guard.as_mut().ok_or(gst::FlowError::NotNegotiated)?; + + let in_frame = + gst_video::VideoFrameRef::from_buffer_ref_readable(inbuf.as_ref(), &state.in_info) + .map_err(|_| { + gst::element_error!( + element, + gst::CoreError::Failed, + ["Failed to map input buffer readable"] + ); + gst::FlowError::Error + })?; + + // And now map the output buffer writable, so we can fill it. + let mut out_frame = + gst_video::VideoFrameRef::from_buffer_ref_writable(outbuf, &state.out_info).map_err( + |_| { + gst::element_error!( + element, + gst::CoreError::Failed, + ["Failed to map output buffer writable"] + ); + gst::FlowError::Error + }, + )?; + + // Keep the various metadata we need for working with the video frames in + // local variables. This saves some typing below. + let width = in_frame.width() as usize; + let in_stride = in_frame.plane_stride()[0] as usize; + let in_data = in_frame.plane_data(0).unwrap(); + let out_stride = out_frame.plane_stride()[0] as usize; + let out_data = out_frame.plane_data_mut(0).unwrap(); + + assert_eq!(in_data.len() % 4, 0); + assert_eq!(out_data.len() / out_stride, in_data.len() / in_stride); + + let in_line_bytes = width * 4; + let out_line_bytes = width * 4; + + assert!(in_line_bytes <= in_stride); + assert!(out_line_bytes <= out_stride); + + for (in_line, out_line) in in_data + .chunks_exact(in_stride) + .zip(out_data.chunks_exact_mut(out_stride)) + { + for (in_p, out_p) in in_line[..in_line_bytes] + .chunks_exact(4) + .zip(out_line[..out_line_bytes].chunks_exact_mut(4)) + { + assert_eq!(out_p.len(), 4); + let hsv = + hsvutils::from_rgb(in_p[..3].try_into().expect("slice with incorrect length")); + + // We handle hue being circular here + let ref_hue_offset = 180.0 - settings.hue_ref; + let mut shifted_hue = hsv[0] + ref_hue_offset; + + if shifted_hue < 0.0 { + shifted_hue += 360.0; + } + + shifted_hue %= 360.0; + + out_p[..3].copy_from_slice(&in_p[..3]); + + out_p[3] = if (shifted_hue - 180.0).abs() <= settings.hue_var + && (hsv[1] - settings.saturation_ref).abs() <= settings.saturation_var + && (hsv[2] - settings.value_ref).abs() <= settings.value_var + { + 255 + } else { + 0 + }; + } + } + + Ok(gst::FlowSuccess::Ok) + } +} diff --git a/video/hsv/src/hsvdetector/mod.rs b/video/hsv/src/hsvdetector/mod.rs new file mode 100644 index 00000000..65c4ddfa --- /dev/null +++ b/video/hsv/src/hsvdetector/mod.rs @@ -0,0 +1,29 @@ +// Copyright (C) 2020 Julien Bardagi +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use glib::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct HsvDetector(ObjectSubclass) @extends gst_base::BaseTransform, gst::Element, gst::Object; +} + +// GStreamer elements need to be thread-safe. For the private implementation this is automatically +// enforced but for the public wrapper type we need to specify this manually. +unsafe impl Send for HsvDetector {} +unsafe impl Sync for HsvDetector {} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "hsvdetector", + gst::Rank::None, + HsvDetector::static_type(), + ) +} diff --git a/video/hsv/src/hsvfilter/imp.rs b/video/hsv/src/hsvfilter/imp.rs new file mode 100644 index 00000000..3329f3cb --- /dev/null +++ b/video/hsv/src/hsvfilter/imp.rs @@ -0,0 +1,410 @@ +// Copyright (C) 2020 Julien Bardagi +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use glib::subclass; +use glib::subclass::prelude::*; +use gst::prelude::*; +use gst::subclass::prelude::*; +use gst::{gst_debug, gst_info}; +use gst_base::subclass::prelude::*; + +use atomic_refcell::AtomicRefCell; +use std::i32; +use std::sync::Mutex; + +use once_cell::sync::Lazy; +use std::convert::TryInto; + +use super::super::hsvutils; + +// Default values of properties +const DEFAULT_HUE_SHIFT: f32 = 0.0; +const DEFAULT_SATURATION_MUL: f32 = 1.0; +const DEFAULT_SATURATION_OFF: f32 = 0.0; +const DEFAULT_VALUE_MUL: f32 = 1.0; +const DEFAULT_VALUE_OFF: f32 = 0.0; + +// Property value storage +#[derive(Debug, Clone, Copy)] +struct Settings { + hue_shift: f32, + saturation_mul: f32, + saturation_off: f32, + value_mul: f32, + value_off: f32, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + hue_shift: DEFAULT_HUE_SHIFT, + saturation_mul: DEFAULT_SATURATION_MUL, + saturation_off: DEFAULT_SATURATION_OFF, + value_mul: DEFAULT_VALUE_MUL, + value_off: DEFAULT_VALUE_OFF, + } + } +} + +// Metadata for the properties +static PROPERTIES: [subclass::Property; 5] = [ + subclass::Property("hue-shift", |name| { + glib::ParamSpec::float( + name, + "Hue shift", + "Hue shifting in degrees", + f32::MIN, + f32::MAX, + DEFAULT_HUE_SHIFT, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("saturation-mul", |name| { + glib::ParamSpec::float( + name, + "Saturation multiplier", + "Saturation multiplier to apply to the saturation value (before offset)", + f32::MIN, + f32::MAX, + DEFAULT_SATURATION_MUL, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("saturation-off", |name| { + glib::ParamSpec::float( + name, + "Saturation offset", + "Saturation offset to add to the saturation value (after multiplier)", + f32::MIN, + f32::MAX, + DEFAULT_SATURATION_OFF, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("value-mul", |name| { + glib::ParamSpec::float( + name, + "Value multiplier", + "Value multiplier to apply to the value (before offset)", + f32::MIN, + f32::MAX, + DEFAULT_VALUE_MUL, + glib::ParamFlags::READWRITE, + ) + }), + subclass::Property("value-off", |name| { + glib::ParamSpec::float( + name, + "Value offset", + "Value offset to add to the value (after multiplier)", + f32::MIN, + f32::MAX, + DEFAULT_VALUE_OFF, + glib::ParamFlags::READWRITE, + ) + }), +]; + +// Stream-specific state, i.e. video format configuration +struct State { + info: gst_video::VideoInfo, +} + +// Struct containing all the element data +pub struct HsvFilter { + settings: Mutex, + state: AtomicRefCell>, +} + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "hsvfilter", + gst::DebugColorFlags::empty(), + Some("Rust HSV transformation filter"), + ) +}); + +impl ObjectSubclass for HsvFilter { + const NAME: &'static str = "HsvFilter"; + type Type = super::HsvFilter; + type ParentType = gst_base::BaseTransform; + type Instance = gst::subclass::ElementInstanceStruct; + type Class = subclass::simple::ClassStruct; + + // Boilerplate macro + glib::object_subclass!(); + + // Creates a new instance. + fn new() -> Self { + Self { + settings: Mutex::new(Default::default()), + state: AtomicRefCell::new(None), + } + } + + fn class_init(klass: &mut Self::Class) { + klass.set_metadata( + "HSV filter", + "Filter/Effect/Converter/Video", + "Works within the HSV colorspace to apply tranformations to incoming frames", + "Julien Bardagi ", + ); + + // src pad capabilities + let caps = gst::Caps::new_simple( + "video/x-raw", + &[ + ( + "format", + &gst::List::new(&[&gst_video::VideoFormat::Rgbx.to_str()]), + ), + ("width", &gst::IntRange::::new(0, i32::MAX)), + ("height", &gst::IntRange::::new(0, i32::MAX)), + ( + "framerate", + &gst::FractionRange::new( + gst::Fraction::new(0, 1), + gst::Fraction::new(i32::MAX, 1), + ), + ), + ], + ); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + klass.add_pad_template(src_pad_template); + + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .unwrap(); + klass.add_pad_template(sink_pad_template); + + // Install all our properties + klass.install_properties(&PROPERTIES); + + klass.configure( + gst_base::subclass::BaseTransformMode::AlwaysInPlace, + false, + false, + ); + } +} + +impl ObjectImpl for HsvFilter { + fn set_property(&self, obj: &Self::Type, id: usize, value: &glib::Value) { + let prop = &PROPERTIES[id]; + + match *prop { + subclass::Property("hue-shift", ..) => { + let mut settings = self.settings.lock().unwrap(); + let hue_shift = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing hue-shift from {} to {}", + settings.hue_shift, + hue_shift + ); + settings.hue_shift = hue_shift; + } + subclass::Property("saturation-mul", ..) => { + let mut settings = self.settings.lock().unwrap(); + let saturation_mul = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing saturation-mul from {} to {}", + settings.saturation_mul, + saturation_mul + ); + settings.saturation_mul = saturation_mul; + } + subclass::Property("saturation-off", ..) => { + let mut settings = self.settings.lock().unwrap(); + let saturation_off = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing saturation-off from {} to {}", + settings.saturation_off, + saturation_off + ); + settings.saturation_off = saturation_off; + } + subclass::Property("value-mul", ..) => { + let mut settings = self.settings.lock().unwrap(); + let value_mul = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing value-mul from {} to {}", + settings.value_mul, + value_mul + ); + settings.value_mul = value_mul; + } + subclass::Property("value-off", ..) => { + let mut settings = self.settings.lock().unwrap(); + let value_off = value.get_some().expect("type checked upstream"); + gst_info!( + CAT, + obj: obj, + "Changing value-off from {} to {}", + settings.value_off, + value_off + ); + settings.value_off = value_off; + } + _ => unimplemented!(), + } + } + + // Called whenever a value of a property is read. It can be called + // at any time from any thread. + fn get_property(&self, _obj: &Self::Type, id: usize) -> glib::Value { + let prop = &PROPERTIES[id]; + + match *prop { + subclass::Property("hue-shift", ..) => { + let settings = self.settings.lock().unwrap(); + settings.hue_shift.to_value() + } + subclass::Property("saturation-mul", ..) => { + let settings = self.settings.lock().unwrap(); + settings.saturation_mul.to_value() + } + subclass::Property("saturation-off", ..) => { + let settings = self.settings.lock().unwrap(); + settings.saturation_off.to_value() + } + subclass::Property("value-mul", ..) => { + let settings = self.settings.lock().unwrap(); + settings.value_mul.to_value() + } + subclass::Property("value-off", ..) => { + let settings = self.settings.lock().unwrap(); + settings.value_off.to_value() + } + _ => unimplemented!(), + } + } +} + +impl ElementImpl for HsvFilter {} + +impl BaseTransformImpl for HsvFilter { + fn get_unit_size(&self, _element: &Self::Type, caps: &gst::Caps) -> Option { + gst_video::VideoInfo::from_caps(caps) + .map(|info| info.size()) + .ok() + } + + fn set_caps( + &self, + element: &Self::Type, + incaps: &gst::Caps, + outcaps: &gst::Caps, + ) -> Result<(), gst::LoggableError> { + let _in_info = match gst_video::VideoInfo::from_caps(incaps) { + Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse input caps")), + Ok(info) => info, + }; + let out_info = match gst_video::VideoInfo::from_caps(outcaps) { + Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse output caps")), + Ok(info) => info, + }; + + gst_debug!( + CAT, + obj: element, + "Configured for caps {} to {}", + incaps, + outcaps + ); + + *self.state.borrow_mut() = Some(State { info: out_info }); + + Ok(()) + } + + fn stop(&self, element: &Self::Type) -> Result<(), gst::ErrorMessage> { + // Drop state + *self.state.borrow_mut() = None; + + gst_info!(CAT, obj: element, "Stopped"); + + Ok(()) + } + + fn transform_ip( + &self, + element: &Self::Type, + buf: &mut gst::BufferRef, + ) -> Result { + let settings = *self.settings.lock().unwrap(); + + let mut state_guard = self.state.borrow_mut(); + let state = state_guard.as_mut().ok_or(gst::FlowError::NotNegotiated)?; + + let mut frame = gst_video::VideoFrameRef::from_buffer_ref_writable(buf, &state.info) + .map_err(|_| { + gst::element_error!( + element, + gst::CoreError::Failed, + ["Failed to map output buffer writable"] + ); + gst::FlowError::Error + })?; + + let width = frame.width() as usize; + let stride = frame.plane_stride()[0] as usize; + let format = frame.format(); + let data = frame.plane_data_mut(0).unwrap(); + + assert_eq!(format, gst_video::VideoFormat::Rgbx); + assert_eq!(data.len() % 4, 0); + + let line_bytes = width * 4; + + for line in data.chunks_exact_mut(stride) { + for p in line[..line_bytes].chunks_exact_mut(4) { + assert_eq!(p.len(), 4); + + let mut hsv = + hsvutils::from_rgb(p[..3].try_into().expect("slice with incorrect length")); + hsv[0] = (hsv[0] + settings.hue_shift) % 360.0; + if hsv[0] < 0.0 { + hsv[0] += 360.0; + } + hsv[1] = hsvutils::Clamp::clamp( + settings.saturation_mul * hsv[1] + settings.saturation_off, + 0.0, + 1.0, + ); + hsv[2] = hsvutils::Clamp::clamp( + settings.value_mul * hsv[2] + settings.value_off, + 0.0, + 1.0, + ); + + p[..3].copy_from_slice(&hsvutils::to_rgb(&hsv)); + } + } + + Ok(gst::FlowSuccess::Ok) + } +} diff --git a/video/hsv/src/hsvfilter/mod.rs b/video/hsv/src/hsvfilter/mod.rs new file mode 100644 index 00000000..720ed86c --- /dev/null +++ b/video/hsv/src/hsvfilter/mod.rs @@ -0,0 +1,29 @@ +// Copyright (C) 2020 Julien Bardagi +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use glib::prelude::*; + +mod imp; + +glib::wrapper! { + pub struct HsvFilter(ObjectSubclass) @extends gst_base::BaseTransform, gst::Element, gst::Object; +} + +// GStreamer elements need to be thread-safe. For the private implementation this is automatically +// enforced but for the public wrapper type we need to specify this manually. +unsafe impl Send for HsvFilter {} +unsafe impl Sync for HsvFilter {} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "hsvfilter", + gst::Rank::None, + HsvFilter::static_type(), + ) +} diff --git a/video/hsv/src/hsvutils.rs b/video/hsv/src/hsvutils.rs new file mode 100644 index 00000000..29479f68 --- /dev/null +++ b/video/hsv/src/hsvutils.rs @@ -0,0 +1,173 @@ +// Copyright (C) 2020 Julien Bardagi +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// Reference used for implementation: https://en.wikipedia.org/wiki/HSL_and_HSV + +#![allow(unstable_name_collisions)] + +// Since the standard 'clamp' feature is still considered unstable, we provide +// a subsititute implementaion here so we can still build with the stable toolchain. +// Source: https://github.com/rust-lang/rust/issues/44095#issuecomment-624879262 +pub trait Clamp: Sized { + fn clamp(self, lower: L, upper: U) -> Self + where + L: Into>, + U: Into>; +} + +impl Clamp for f32 { + fn clamp(self, lower: L, upper: U) -> Self + where + L: Into>, + U: Into>, + { + let below = match lower.into() { + None => self, + Some(lower) => self.max(lower), + }; + match upper.into() { + None => below, + Some(upper) => below.min(upper), + } + } +} + +const EPSILON: f32 = 0.00001; + +// Converts a RGB pixel to HSV +#[inline] +pub fn from_rgb(in_p: &[u8; 3]) -> [f32; 3] { + let r = in_p[0] as f32 / 255.0; + let g = in_p[1] as f32 / 255.0; + let b = in_p[2] as f32 / 255.0; + + let value: f32 = *in_p + .iter() + .max() + .expect("Cannot find max value from rgb input") as f32 + / 255.0; + let chroma: f32 = value + - (*in_p + .iter() + .min() + .expect("Cannot find min value from rgb input") as f32 + / 255.0); + + let mut hue: f32 = if chroma == 0.0 { + 0.0 + } else if (value - r).abs() < EPSILON { + 60.0 * ((g - b) / chroma) + } else if (value - g).abs() < EPSILON { + 60.0 * (2.0 + ((b - r) / chroma)) + } else if (value - b).abs() < EPSILON { + 60.0 * (4.0 + ((r - g) / chroma)) + } else { + 0.0 + }; + + if hue < 0.0 { + hue += 360.0; + } + + let saturation: f32 = if value == 0.0 { 0.0 } else { chroma / value }; + + [ + hue % 360.0, + saturation.clamp(0.0, 1.0), + value.clamp(0.0, 1.0), + ] +} + +// Converts a HSV pixel to RGB +#[inline] +pub fn to_rgb(in_p: &[f32; 3]) -> [u8; 3] { + let c: f32 = in_p[2] * in_p[1]; + let hue_prime: f32 = in_p[0] / 60.0; + + let x: f32 = c * (1.0 - ((hue_prime % 2.0) - 1.0).abs()); + + let rgb_prime = if hue_prime < 0.0 { + [0.0, 0.0, 0.0] + } else if hue_prime <= 1.0 { + [c, x, 0.0] + } else if hue_prime <= 2.0 { + [x, c, 0.0] + } else if hue_prime <= 3.0 { + [0.0, c, x] + } else if hue_prime <= 4.0 { + [0.0, x, c] + } else if hue_prime <= 5.0 { + [x, 0.0, c] + } else if hue_prime <= 6.0 { + [c, 0.0, x] + } else { + [0.0, 0.0, 0.0] + }; + + let m = in_p[2] - c; + + [ + ((rgb_prime[0] + m) * 255.0).clamp(0.0, 255.0) as u8, + ((rgb_prime[1] + m) * 255.0).clamp(0.0, 255.0) as u8, + ((rgb_prime[2] + m) * 255.0).clamp(0.0, 255.0) as u8, + ] +} + +#[cfg(test)] +mod tests { + + fn is_equivalent(hsv: &[f32; 3], expected: &[f32; 3], eps: f32) -> bool { + // We handle hue being circular here + let ref_hue_offset = 180.0 - expected[0]; + let mut shifted_hue = hsv[0] + ref_hue_offset; + + if shifted_hue < 0.0 { + shifted_hue += 360.0; + } + + shifted_hue %= 360.0; + + (shifted_hue - 180.0).abs() < eps + && (hsv[1] - expected[1]).abs() < eps + && (hsv[2] - expected[2]).abs() < eps + } + + const RGB_WHITE: [u8; 3] = [255, 255, 255]; + const RGB_BLACK: [u8; 3] = [0, 0, 0]; + const RGB_RED: [u8; 3] = [255, 0, 0]; + const RGB_GREEN: [u8; 3] = [0, 255, 0]; + const RGB_BLUE: [u8; 3] = [0, 0, 255]; + + const HSV_WHITE: [f32; 3] = [0.0, 0.0, 1.0]; + const HSV_BLACK: [f32; 3] = [0.0, 0.0, 0.0]; + const HSV_RED: [f32; 3] = [0.0, 1.0, 1.0]; + const HSV_GREEN: [f32; 3] = [120.0, 1.0, 1.0]; + const HSV_BLUE: [f32; 3] = [240.0, 1.0, 1.0]; + + #[test] + fn test_from_rgb() { + use super::*; + + assert!(is_equivalent(&from_rgb(&RGB_WHITE), &HSV_WHITE, EPSILON)); + assert!(is_equivalent(&from_rgb(&RGB_BLACK), &HSV_BLACK, EPSILON)); + assert!(is_equivalent(&from_rgb(&RGB_RED), &HSV_RED, EPSILON)); + assert!(is_equivalent(&from_rgb(&RGB_GREEN), &HSV_GREEN, EPSILON)); + assert!(is_equivalent(&from_rgb(&RGB_BLUE), &HSV_BLUE, EPSILON)); + } + + #[test] + fn test_to_rgb() { + use super::*; + + assert!(to_rgb(&HSV_WHITE) == RGB_WHITE); + assert!(to_rgb(&HSV_BLACK) == RGB_BLACK); + assert!(to_rgb(&HSV_RED) == RGB_RED); + assert!(to_rgb(&HSV_GREEN) == RGB_GREEN); + assert!(to_rgb(&HSV_BLUE) == RGB_BLUE); + } +} diff --git a/video/hsv/src/lib.rs b/video/hsv/src/lib.rs new file mode 100644 index 00000000..d78efdb3 --- /dev/null +++ b/video/hsv/src/lib.rs @@ -0,0 +1,29 @@ +// Copyright (C) 2020 Julien Bardagi +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +mod hsvdetector; +mod hsvfilter; +mod hsvutils; + +fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + hsvfilter::register(plugin)?; + hsvdetector::register(plugin)?; + Ok(()) +} + +gst::plugin_define!( + hsv, + env!("CARGO_PKG_DESCRIPTION"), + plugin_init, + concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")), + "MIT/X11", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_REPOSITORY"), + env!("BUILD_REL_DATE") +);