From 33b7959fab09759fce06cc5658700814c6c27713 Mon Sep 17 00:00:00 2001 From: Rafael Caricio Date: Sun, 27 Mar 2022 11:31:47 +0200 Subject: [PATCH] Initial commit --- .gitignore | 1 + Cargo.lock | 1143 ++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 47 ++ README.md | 5 + build.rs | 3 + src/lib.rs | 29 + src/transcriber/imp.rs | 1042 ++++++++++++++++++++++++++++++++++++ src/transcriber/mod.rs | 28 + src/vosk_client/mod.rs | 35 ++ 9 files changed, 2333 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 build.rs create mode 100644 src/lib.rs create mode 100644 src/transcriber/imp.rs create mode 100644 src/transcriber/mod.rs create mode 100644 src/vosk_client/mod.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..a5cc637 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1143 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" + +[[package]] +name = "async-tungstenite" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b71b31561643aa8e7df3effe284fa83ab1a840e52294c5f4bd7bfd8b2becbb" +dependencies = [ + "futures-io", + "futures-util", + "log", + "native-tls", + "pin-project-lite", + "tokio", + "tokio-native-tls", + "tungstenite", +] + +[[package]] +name = "atomic_refcell" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b5e5f48b927f04e952dedc932f31995a65a0bf65ec971c74436e51bf6e970d" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-expr" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e068cb2806bbc15b439846dc16c5f89f8599f2c3e4d73d4449d38f9b2f0b6c5" +dependencies = [ + "smallvec", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "winapi", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-executor" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" + +[[package]] +name = "futures-task" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" + +[[package]] +name = "futures-util" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "glib" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a826fad715b57834920839d7a594c3b5e416358c7d790bdaba847a40d7c1d96d" +dependencies = [ + "bitflags", + "futures-channel", + "futures-core", + "futures-executor", + "futures-task", + "glib-macros", + "glib-sys", + "gobject-sys", + "libc", + "once_cell", + "smallvec", + "thiserror", +] + +[[package]] +name = "glib-macros" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac4d47c544af67747652ab1865ace0ffa1155709723ac4f32e97587dd4735b2" +dependencies = [ + "anyhow", + "heck", + "proc-macro-crate", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "glib-sys" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4b192f8e65e9cf76cbf4ea71fa8e3be4a0e18ffe3d68b8da6836974cc5bad4" +dependencies = [ + "libc", + "system-deps", +] + +[[package]] +name = "gobject-sys" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d57ce44246becd17153bd035ab4d32cfee096a657fc01f2231c9278378d1e0a" +dependencies = [ + "glib-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gst-plugin-version-helper" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6a4dd1cb931cc6b49af354a68f21b3aee46b5b07370215d942f3a71542123f" +dependencies = [ + "chrono", +] + +[[package]] +name = "gst-plugin-vosk" +version = "0.1.0" +dependencies = [ + "async-tungstenite", + "atomic_refcell", + "futures", + "gst-plugin-version-helper", + "gstreamer", + "gstreamer-base", + "once_cell", + "serde", + "serde_derive", + "serde_json", + "tokio", +] + +[[package]] +name = "gstreamer" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1545fac08d7a28f8707101298cbf99d1bc72529698ff2d1fec87cc30a3fb9a" +dependencies = [ + "bitflags", + "cfg-if", + "futures-channel", + "futures-core", + "futures-util", + "glib", + "gstreamer-sys", + "libc", + "muldiv", + "num-integer", + "num-rational", + "once_cell", + "option-operations", + "paste", + "pretty-hex", + "thiserror", +] + +[[package]] +name = "gstreamer-base" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224f35f36582407caf58ded74854526beeecc23d0cf64b8d1c3e00584ed6863f" +dependencies = [ + "bitflags", + "cfg-if", + "glib", + "gstreamer", + "gstreamer-base-sys", + "libc", +] + +[[package]] +name = "gstreamer-base-sys" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a083493c3c340e71fa7c66eebda016e9fafc03eb1b4804cf9b2bad61994b078e" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sys" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3517a65d3c2e6f8905b456eba5d53bda158d664863aef960b44f651cb7d33e2" +dependencies = [ + "glib-sys", + "gobject-sys", + "libc", + "system-deps", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" + +[[package]] +name = "log" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "mio" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "muldiv" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5136edda114182728ccdedb9f5eda882781f35fa6e80cc360af12a8932507f3" + +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" + +[[package]] +name = "openssl" +version = "0.10.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-operations" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95d6113415f41b268f1195907427519769e40ee6f28cbb053795098a2c16f447" +dependencies = [ + "paste", +] + +[[package]] +name = "paste" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project-lite" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "pretty-hex" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5c99d529f0d30937f6f4b8a86d988047327bb88d04d2c4afc356de74722131" + +[[package]] +name = "proc-macro-crate" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +dependencies = [ + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" +dependencies = [ + "bitflags", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "security-framework" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" + +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" + +[[package]] +name = "smallvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" + +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "system-deps" +version = "6.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a45a1c4c9015217e12347f2a411b57ce2c4fc543913b14b6fe40483328e709" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +dependencies = [ + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "winapi", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "tungstenite" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96a2dea40e7570482f28eb57afbe42d97551905da6a9400acc5c328d24004f5" +dependencies = [ + "base64", + "byteorder", + "bytes", + "http", + "httparse", + "log", + "native-tls", + "rand", + "sha-1", + "thiserror", + "url", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version-compare" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe88247b92c1df6b6de80ddc290f3976dbdf2f5f5d3fd049a9fb598c6dd5ca73" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..f89992a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "gst-plugin-vosk" +version = "0.1.0" +edition = "2021" +authors = ["Rafael Caricio "] +repository = "" +license = "MPL-2.0" +description = "plugin" +build = "build.rs" + +[dependencies] +gst = { package = "gstreamer", version = "0.18" } +gstreamer-base = "0.18" +once_cell = "1" +atomic_refcell = "0.1" +serde = "1" +serde_derive = "1" +serde_json = "1" +futures = "0.3" +tokio = { version = "1.0", features = [ "rt-multi-thread", "time" ] } +async-tungstenite = { version = "0.17", features = ["tokio", "tokio-runtime", "tokio-native-tls"] } + +[build-dependencies] +gst-plugin-version-helper = "0.7.3" + +[lib] +name = "gstvosk" +crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" + +[features] +# GStreamer 1.14 is required for static linking +static = ["gst/v1_14"] +capi = [] + +[package.metadata.capi] +min_version = "0.8.0" + +[package.metadata.capi.header] +enabled = false + +[package.metadata.capi.library] +install_subdir = "gstreamer-1.0" +versioning = false + +[package.metadata.capi.pkg_config] +requires_private = "gstreamer-1.0, gstreamer-base-1.0, gobject-2.0, glib-2.0, gmodule-2.0" diff --git a/README.md b/README.md new file mode 100644 index 0000000..a563599 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +Vosk Speech Recognition GStreamer Plugin +======================================== + +Transcription of speech using [Vosk Toolkit](https://alphacephei.com/vosk/). Can be used to generate subtitles for +videos, transcription of audio notes, etc. diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..cda12e5 --- /dev/null +++ b/build.rs @@ -0,0 +1,3 @@ +fn main() { + gst_plugin_version_helper::info() +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..76a91bc --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,29 @@ +// Copyright (C) 2022 Rafael Caricio +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; + +mod transcriber; +mod vosk_client; + +fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + transcriber::register(plugin)?; + Ok(()) +} + +gst::plugin_define!( + vosktranscriber, + env!("CARGO_PKG_DESCRIPTION"), + plugin_init, + concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")), + "MPL", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_REPOSITORY"), + env!("BUILD_REL_DATE") +); \ No newline at end of file diff --git a/src/transcriber/imp.rs b/src/transcriber/imp.rs new file mode 100644 index 0000000..c9b9e9a --- /dev/null +++ b/src/transcriber/imp.rs @@ -0,0 +1,1042 @@ +// Copyright (C) 2022 Rafael Caricio +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use crate::vosk_client::{Configuration, Transcript, WordInfo}; +use async_tungstenite::tungstenite::error::Error as WsError; +use async_tungstenite::{tokio::connect_async, tungstenite::Message}; +use atomic_refcell::AtomicRefCell; +use futures::channel::mpsc; +use futures::future::{abortable, AbortHandle}; +use futures::prelude::*; +use futures::Sink; +use gst::glib; +use gst::prelude::*; +use gst::subclass::prelude::*; +use gst::{ + element_error, error_msg, gst_debug, gst_error, gst_info, gst_log, gst_trace, gst_warning, + loggable_error, +}; +use once_cell::sync::Lazy; +use std::cmp::Ordering; +use std::collections::VecDeque; +use std::pin::Pin; +use std::sync::Mutex; +use tokio::runtime; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "vosk_transcriber", + gst::DebugColorFlags::empty(), + Some("Vosk transcription element"), + ) +}); + +static RUNTIME: Lazy = Lazy::new(|| { + runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(1) + .build() + .unwrap() +}); + +const DEFAULT_LATENCY: gst::ClockTime = gst::ClockTime::from_seconds(5); +const DEFAULT_SERVER_ADDRESS: &str = "ws://localhost:2700"; +const GRANULARITY: gst::ClockTime = gst::ClockTime::from_mseconds(100); + +#[derive(Debug, Clone)] +struct Settings { + /// Total time to allow for transcribing audio. How late the buffer we produce will be. + latency: gst::ClockTime, + + /// The address of the gRPC server to connect to for transcription. + server_address: String, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + latency: DEFAULT_LATENCY, + server_address: DEFAULT_SERVER_ADDRESS.to_string(), + } + } +} + +struct State { + /// Flag to indicate if we are connected to the remote Vosk server. + connected: bool, + + /// The current queued buffers to be processed in the next iteration + buffers: VecDeque, + + // The time we started processing the buffers + start_time: Option, + + /// Flag to indicate that we need to send the required stream initialization events to the src pad + send_events: bool, + + /// Flag to indicate that we need to send the EOS event to the src pad + send_eos: bool, + + /// Flag to indicate that we need to send a discontinuity buffer in the src pad + send_discontinuity: bool, + + /// The segment of the stream we are receiving buffers from (sink pad) + in_segment: gst::FormattedSegment, + + /// The sequence number of the segment we are receiving buffers from (sink pad) + seqnum: gst::Seqnum, + + /// Track the segment position of the stream we are sending in the src pad + out_segment: gst::FormattedSegment, + + /// The channel to send messages to the Vosk server transmission task + sender: Option>, + + /// Handle to stop the continuous reception of messages from the Vosk server + recv_abort_handle: Option, + + /// Handler to stop the continuous transmission of messages to the Vosk server + send_abort_handle: Option, +} + +impl Default for State { + fn default() -> Self { + Self { + connected: false, + buffers: VecDeque::new(), + start_time: None, + send_events: true, + send_eos: false, + send_discontinuity: true, + in_segment: gst::FormattedSegment::new(), + seqnum: gst::Seqnum::next(), + out_segment: gst::FormattedSegment::new(), + sender: None, + recv_abort_handle: None, + send_abort_handle: None, + } + } +} + +type WsSink = Pin + Send + Sync>>; + +pub struct Transcriber { + /// Pad that we will receive the audio buffers from + srcpad: gst::Pad, + + /// Pad that we will send the text transcription buffers to + sinkpad: gst::Pad, + + // The settings of the element + settings: Mutex, + + // The state of the element, it is resent every time the element restarts + state: Mutex, + + // The sink of the messages to the Vosk server + ws_sink: AtomicRefCell>, +} + +impl Transcriber { + fn dequeue(&self, element: &super::Transcriber) -> bool { + let mut items = vec![]; + + let now = match element.current_running_time() { + Some(now) => now, + None => { + return true; + } + }; + + let latency = self.settings.lock().unwrap().latency; + + let mut state = self.state.lock().unwrap(); + + if state.start_time.is_none() { + state.start_time = Some(now); + state.out_segment.set_position(now); + } + + let start_time = state.start_time.unwrap(); + let mut last_position = state.out_segment.position().unwrap(); + + let send_eos = state.send_eos && state.buffers.is_empty(); + + while let Some(buf) = state.buffers.front() { + let pts = buf.pts().unwrap(); + gst_trace!( + CAT, + obj: element, + "Checking now {} if item is ready for dequeuing, PTS {}, threshold {} vs {}", + now, + pts, + pts + latency.saturating_sub(3 * GRANULARITY), + now - start_time + ); + + // If buffer pts is from the time that has passed so far, we add it to be dequeued + if pts + latency.saturating_sub(3 * GRANULARITY) < now - start_time { + /* Safe unwrap, we know we have an item */ + let mut buf = state.buffers.pop_front().unwrap(); + + { + let buf_mut = buf.get_mut().unwrap(); + + // Fixes the buffer's presentation time + buf_mut.set_pts(start_time + pts); + } + + items.push(buf); + } else { + break; + } + } + + let seqnum = state.seqnum; + + drop(state); + + // We're EOS, we can pause and exit early + if send_eos { + let _ = self.srcpad.pause_task(); + + return self + .srcpad + .push_event(gst::event::Eos::builder().seqnum(seqnum).build()); + } + + for mut buf in items.drain(..) { + let mut pts = buf.pts().unwrap(); + let mut duration = buf.duration().unwrap(); + + match pts.cmp(&last_position) { + Ordering::Greater => { + // When the PTS is greater than the last position, we need to send a gap event + let gap_event = gst::event::Gap::builder(last_position) + .duration(pts - last_position) + .seqnum(seqnum) + .build(); + gst_log!(CAT, "Pushing gap: {} -> {}", last_position, pts); + if !self.srcpad.push_event(gap_event) { + return false; + } + } + Ordering::Less => { + let delta = last_position - pts; + + gst_warning!( + CAT, + obj: element, + "Updating item PTS ({} < {}), consider increasing latency", + pts, + last_position + ); + + pts = last_position; + duration = duration.saturating_sub(delta); + + { + let buf_mut = buf.get_mut().unwrap(); + + buf_mut.set_pts(pts); + buf_mut.set_duration(duration); + } + } + _ => (), + } + + last_position = pts + duration; + + gst_debug!(CAT, "Pushing buffer: {} -> {}", pts, pts + duration); + + if self.srcpad.push(buf).is_err() { + return false; + } + } + + /* next, push a gap if we're lagging behind the target position */ + gst_trace!( + CAT, + obj: element, + "Checking now: {} if we need to push a gap, last_position: {}, threshold: {}", + now, + last_position, + last_position + latency.saturating_sub(GRANULARITY) + ); + + if now > last_position + latency.saturating_sub(GRANULARITY) { + let duration = now - last_position - latency.saturating_sub(GRANULARITY); + + let gap_event = gst::event::Gap::builder(last_position) + .duration(duration) + .seqnum(seqnum) + .build(); + + gst_log!( + CAT, + "Pushing gap: {} -> {}", + last_position, + last_position + duration + ); + + last_position += duration; + + if !self.srcpad.push_event(gap_event) { + return false; + } + } + + self.state + .lock() + .unwrap() + .out_segment + .set_position(last_position); + + true + } + + fn enqueue( + &self, + element: &super::Transcriber, + state: &mut State, + transcription: &Vec, + ) { + for item in transcription.iter() { + let start_time = gst::ClockTime::from_nseconds((item.start * 1_000_000_000.0) as u64); + let end_time = gst::ClockTime::from_nseconds((item.end * 1_000_000_000.0) as u64); + + // Should be sent now + gst_debug!( + CAT, + obj: element, + "Item is ready for queuing: {}, PTS {}", + item.word, + start_time + ); + + let mut buf = gst::Buffer::from_mut_slice(item.word.clone().into_bytes()); + { + let buf = buf.get_mut().unwrap(); + + if state.send_discontinuity { + buf.set_flags(gst::BufferFlags::DISCONT); + state.send_discontinuity = false; + } + + // The presentation time here is from the start of the media, not from the start of the stream + buf.set_pts(start_time); + buf.set_duration(end_time - start_time); + } + + state.buffers.push_back(buf); + } + } + + fn loop_fn( + &self, + element: &super::Transcriber, + receiver: &mut mpsc::Receiver, + ) -> Result<(), gst::ErrorMessage> { + let mut events = { + let mut events = vec![]; + + let mut state = self.state.lock().unwrap(); + + // Events that are always sent when starting a stream on gst elements + if state.send_events { + events.push( + gst::event::StreamStart::builder("transcription") + .seqnum(state.seqnum) + .build(), + ); + + let caps = gst::Caps::builder("text/x-raw") + .field("format", "utf8") + .build(); + events.push( + gst::event::Caps::builder(&caps) + .seqnum(state.seqnum) + .build(), + ); + + events.push( + gst::event::Segment::builder(&state.out_segment) + .seqnum(state.seqnum) + .build(), + ); + + // Events are sent only once, so we set this to false + state.send_events = false; + } + + events + }; + + for event in events.drain(..) { + gst_info!(CAT, obj: element, "Sending {:?}", event); + self.srcpad.push_event(event); + } + + // Process the responses from Vosk server and produce text buffers + let process_next_server_response_future = async move { + let msg = match receiver.next().await { + Some(msg) => msg, + // Sender was closed so we stop the task + None => { + let _ = self.srcpad.pause_task(); + return Ok(()); + } + }; + + match msg { + Message::Text(payload) => { + gst_trace!(CAT, obj: element, "got payload: {}", payload,); + + let transcript: Transcript = match serde_json::from_str(&payload) { + Ok(transcript) => transcript, + Err(err) => { + // The payload is still not a final transcript, so we just ignore it + return Ok(()); + } + }; + + gst_trace!( + CAT, + obj: element, + "result: {}", + serde_json::to_string_pretty(&transcript.result).unwrap(), + ); + + let mut state = self.state.lock().unwrap(); + + self.enqueue(element, &mut state, &transcript.result); + + Ok(()) + } + + _ => Ok(()), + } + }; + + // Wrap in a timeout so we can push gaps regularly + let timed_call_future = async move { + match tokio::time::timeout(GRANULARITY.into(), process_next_server_response_future) + .await + { + Err(_) => { + if !self.dequeue(element) { + gst_info!(CAT, obj: element, "Failed to push gap event, pausing"); + + let _ = self.srcpad.pause_task(); + } + Ok(()) + } + Ok(res) => { + if !self.dequeue(element) { + gst_info!(CAT, obj: element, "Failed to push gap event, pausing"); + + let _ = self.srcpad.pause_task(); + } + res + } + } + }; + + let _enter = RUNTIME.enter(); + futures::executor::block_on(timed_call_future) + } + + /// Start task that will be called once to initialize the element + fn start_task(&self, element: &super::Transcriber) -> Result<(), gst::LoggableError> { + let (sender, mut receiver) = mpsc::channel(1); + { + let mut state = self.state.lock().unwrap(); + state.sender = Some(sender); + } + + // This task is called repeatedly to produce text buffers and stream events. + let res = self.srcpad.start_task({ + let element_weak = element.downgrade(); + let pad_weak = self.srcpad.downgrade(); + move || { + let element = match element_weak.upgrade() { + Some(element) => element, + None => { + if let Some(pad) = pad_weak.upgrade() { + let _ = pad.pause_task(); + } + return; + } + }; + + let transcribe = element.imp(); + // Do the actual work, of producing buffers and events. + if let Err(err) = transcribe.loop_fn(&element, &mut receiver) { + element_error!( + &element, + gst::StreamError::Failed, + ["Streaming failed: {}", err] + ); + let _ = transcribe.srcpad.pause_task(); + } + } + }); + if res.is_err() { + return Err(loggable_error!(CAT, "Failed to start pad task")); + } + Ok(()) + } + + fn src_activatemode( + &self, + _pad: &gst::Pad, + element: &super::Transcriber, + _mode: gst::PadMode, + active: bool, + ) -> Result<(), gst::LoggableError> { + if active { + self.start_task(element)?; + } else { + { + let mut state = self.state.lock().unwrap(); + state.sender = None; + } + + let _ = self.srcpad.stop_task(); + } + + Ok(()) + } + + fn src_query( + &self, + pad: &gst::Pad, + element: &super::Transcriber, + query: &mut gst::QueryRef, + ) -> bool { + use gst::QueryView; + + gst_log!(CAT, obj: pad, "Handling query {:?}", query); + + match query.view_mut() { + QueryView::Latency(mut q) => { + let mut peer_query = gst::query::Latency::new(); + + let ret = self.sinkpad.peer_query(&mut peer_query); + + // Adds our own latency to the upstream peer's latency + if ret { + let (_, min, _) = peer_query.result(); + let our_latency = self.settings.lock().unwrap().latency; + // We never drop buffers, so our max latency is set to infinity + q.set(true, our_latency + min, gst::ClockTime::NONE); + } + ret + } + QueryView::Position(mut q) => { + if q.format() == gst::Format::Time { + let state = self.state.lock().unwrap(); + q.set( + state + .out_segment + .to_stream_time(state.out_segment.position()), + ); + true + } else { + false + } + } + _ => pad.query_default(Some(element), query), + } + } + + fn sink_event(&self, pad: &gst::Pad, element: &super::Transcriber, event: gst::Event) -> bool { + use gst::EventView; + + gst_log!(CAT, obj: pad, "Handling event {:?}", event); + + match event.view() { + EventView::Eos(_) => match self.handle_buffer(pad, element, None) { + Err(err) => { + gst_error!(CAT, "Failed to send EOS: {}", err); + false + } + Ok(_) => true, + }, + EventView::FlushStart(_) => { + gst_info!(CAT, obj: element, "Received flush start, disconnecting"); + let mut ret = pad.event_default(Some(element), event); + + match self.srcpad.stop_task() { + Err(err) => { + gst_error!(CAT, obj: element, "Failed to stop srcpad task: {}", err); + + self.disconnect(element); + + ret = false; + } + Ok(_) => { + self.disconnect(element); + } + }; + + ret + } + EventView::FlushStop(_) => { + gst_info!(CAT, obj: element, "Received flush stop, restarting task"); + + if pad.event_default(Some(element), event) { + match self.start_task(element) { + Err(err) => { + gst_error!(CAT, obj: element, "Failed to start srcpad task: {}", err); + false + } + Ok(_) => true, + } + } else { + false + } + } + EventView::Segment(e) => { + let segment = match e.segment().clone().downcast::() { + Err(segment) => { + element_error!( + element, + gst::StreamError::Format, + ["Only Time segments supported, got {:?}", segment.format(),] + ); + return false; + } + Ok(segment) => segment, + }; + + let mut state = self.state.lock().unwrap(); + + state.in_segment = segment; + state.seqnum = e.seqnum(); + + true + } + EventView::Tag(_) => true, + EventView::Caps(e) => { + gst_info!(CAT, "Received caps {:?}", e); + true + } + EventView::StreamStart(_) => true, + _ => pad.event_default(Some(element), event), + } + } + + async fn sync_and_send( + &self, + element: &super::Transcriber, + buffer: Option, + ) -> Result { + let mut delay = None; + + { + let state = self.state.lock().unwrap(); + + if let Some(buffer) = &buffer { + let running_time = state.in_segment.to_running_time(buffer.pts()); + let now = element.current_running_time(); + + delay = running_time.opt_checked_sub(now).ok().flatten(); + } + } + + // Wait until now is close enough to the buffer's PTS + if let Some(delay) = delay { + tokio::time::sleep(delay.into()).await; + } + + if let Some(ws_sink) = self.ws_sink.borrow_mut().as_mut() { + if let Some(buffer) = buffer { + let data = buffer.map_readable().unwrap(); + for chunk in data.chunks(8192) { + ws_sink + .send(Message::Binary(chunk.to_vec())) + .await + .map_err(|err| { + gst_error!(CAT, obj: element, "Failed sending packet: {}", err); + gst::FlowError::Error + })?; + } + } else { + // Send end of stream + ws_sink + .send(Message::Text("{\"eof\": 1}".to_string())) + .await + .map_err(|err| { + gst_error!(CAT, obj: element, "Failed sending packet: {}", err); + gst::FlowError::Error + })?; + } + } + + Ok(gst::FlowSuccess::Ok) + } + + fn handle_buffer( + &self, + _pad: &gst::Pad, + element: &super::Transcriber, + buffer: Option, + ) -> Result { + gst_log!(CAT, obj: element, "Handling {:?}", buffer); + + self.ensure_connection(element).map_err(|err| { + element_error!( + element, + gst::StreamError::Failed, + ["Streaming failed: {}", err] + ); + gst::FlowError::Error + })?; + + let (send_handle, abort_handle) = abortable(self.sync_and_send(element, buffer)); + + self.state.lock().unwrap().send_abort_handle = Some(abort_handle); + + let res = { + let _enter = RUNTIME.enter(); + futures::executor::block_on(send_handle) + }; + + match res { + Err(_) => Err(gst::FlowError::Flushing), + Ok(res) => res, + } + } + + fn sink_chain( + &self, + pad: &gst::Pad, + element: &super::Transcriber, + buffer: gst::Buffer, + ) -> Result { + self.handle_buffer(pad, element, Some(buffer)) + } + + fn ensure_connection(&self, element: &super::Transcriber) -> Result<(), gst::ErrorMessage> { + let state = self.state.lock().unwrap(); + + if state.connected { + return Ok(()); + } + + let settings = self.settings.lock().unwrap(); + if settings.latency <= 2 * GRANULARITY { + gst_error!( + CAT, + obj: element, + "latency must be greater than 200 milliseconds" + ); + return Err(error_msg!( + gst::LibraryError::Settings, + ["latency must be greater than 200 milliseconds"] + )); + } + + gst_info!(CAT, obj: element, "Connecting .."); + + let url = settings.server_address.clone(); + + drop(settings); + drop(state); + + // Set up the server to handle the incoming audio sample rate + let in_caps = self.sinkpad.current_caps().unwrap(); + let s = in_caps.structure(0).unwrap(); + let sample_rate = s.get::("rate").unwrap(); + + let (ws, _) = { + let _enter = RUNTIME.enter(); + futures::executor::block_on(connect_async(url)).map_err(|err| { + gst_error!(CAT, obj: element, "Failed to connect: {}", err); + error_msg!(gst::CoreError::Failed, ["Failed to connect: {}", err]) + })? + }; + + let (mut ws_sink, mut ws_stream) = ws.split(); + + let config = Configuration::new(sample_rate); + let packet = serde_json::to_vec(&config).unwrap(); + ws_sink.send(Message::Binary(packet)).await.map_err(|err| { + gst_error!( + CAT, + obj: element, + "Failed to configure Vosk server for the expected sample rate: {}", + err + ); + gst::FlowError::Error + })?; + + *self.ws_sink.borrow_mut() = Some(Box::pin(ws_sink)); + + let element_weak = element.downgrade(); + let recv_handle = async move { + + while let Some(element) = element_weak.upgrade() { + let transcribe = element.imp(); + let msg = match ws_stream.next().await { + Some(msg) => msg, + None => { + let mut state = transcribe.state.lock().unwrap(); + state.send_eos = true; + break; + } + }; + + let msg = match msg { + Ok(msg) => msg, + Err(err) => { + gst_error!(CAT, "Failed to receive data: {}", err); + element_error!( + element, + gst::StreamError::Failed, + ["Streaming failed: {}", err] + ); + break; + } + }; + + let mut sender = transcribe.state.lock().unwrap().sender.clone(); + + if let Some(sender) = sender.as_mut() { + if sender.send(msg).await.is_err() { + break; + } + } + } + }; + + let mut state = self.state.lock().unwrap(); + + let (future, abort_handle) = abortable(recv_handle); + + state.recv_abort_handle = Some(abort_handle); + + RUNTIME.spawn(future); + + state.connected = true; + + gst_info!(CAT, obj: element, "Connected"); + + Ok(()) + } + + fn disconnect(&self, element: &super::Transcriber) { + let mut state = self.state.lock().unwrap(); + + gst_info!(CAT, obj: element, "Unpreparing"); + + if let Some(abort_handle) = state.recv_abort_handle.take() { + abort_handle.abort(); + } + + if let Some(abort_handle) = state.send_abort_handle.take() { + abort_handle.abort(); + } + + *state = State::default(); + + gst_info!( + CAT, + obj: element, + "Unprepared, connected: {}!", + state.connected + ); + } +} + +#[glib::object_subclass] +impl ObjectSubclass for Transcriber { + const NAME: &'static str = "VoskTranscriber"; + type Type = super::Transcriber; + type ParentType = gst::Element; + + fn with_class(klass: &Self::Class) -> Self { + let templ = klass.pad_template("sink").unwrap(); + let sinkpad = gst::Pad::builder_with_template(&templ, Some("sink")) + .chain_function(|pad, parent, buffer| { + Transcriber::catch_panic_pad_function( + parent, + || Err(gst::FlowError::Error), + |transcriber, element| transcriber.sink_chain(pad, element, buffer), + ) + }) + .event_function(|pad, parent, event| { + Transcriber::catch_panic_pad_function( + parent, + || false, + |transcriber, element| transcriber.sink_event(pad, element, event), + ) + }) + .build(); + + let templ = klass.pad_template("src").unwrap(); + let srcpad = gst::Pad::builder_with_template(&templ, Some("src")) + .activatemode_function(|pad, parent, mode, active| { + Transcriber::catch_panic_pad_function( + parent, + || Err(loggable_error!(CAT, "Panic activating src pad with mode")), + |transcriber, element| transcriber.src_activatemode(pad, element, mode, active), + ) + }) + .query_function(|pad, parent, query| { + Transcriber::catch_panic_pad_function( + parent, + || false, + |transcriber, element| transcriber.src_query(pad, element, query), + ) + }) + .flags(gst::PadFlags::FIXED_CAPS) + .build(); + + Self { + srcpad, + sinkpad, + settings: Mutex::new(Settings::default()), + state: Mutex::new(State::default()), + ws_sink: AtomicRefCell::new(None), + } + } +} + +impl ObjectImpl for Transcriber { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![glib::ParamSpecUInt::new( + "latency", + "Latency", + "Amount of milliseconds to allow Vosk to transcribe", + 0, + std::u32::MAX, + DEFAULT_LATENCY.mseconds() as u32, + glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_READY, + )] + }); + + PROPERTIES.as_ref() + } + + fn constructed(&self, obj: &Self::Type) { + self.parent_constructed(obj); + + obj.add_pad(&self.sinkpad).unwrap(); + obj.add_pad(&self.srcpad).unwrap(); + obj.set_element_flags(gst::ElementFlags::PROVIDE_CLOCK | gst::ElementFlags::REQUIRE_CLOCK); + } + + fn set_property( + &self, + _obj: &Self::Type, + _id: usize, + value: &glib::Value, + pspec: &glib::ParamSpec, + ) { + match pspec.name() { + "latency" => { + let mut settings = self.settings.lock().unwrap(); + settings.latency = gst::ClockTime::from_mseconds( + value.get::().expect("type checked upstream").into(), + ); + } + _ => unimplemented!(), + } + } + + fn property(&self, _obj: &Self::Type, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "latency" => { + let settings = self.settings.lock().unwrap(); + (settings.latency.mseconds() as u32).to_value() + } + _ => unimplemented!(), + } + } +} + +impl GstObjectImpl for Transcriber {} + +impl ElementImpl for Transcriber { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "Transcriber", + "Audio/Text/Filter", + "Speech to Text filter, using Vosk toolkit", + "Rafael Caricio ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let src_caps = gst::Caps::builder("text/x-raw") + .field("format", "utf8") + .build(); + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &src_caps, + ) + .unwrap(); + + let sink_caps = gst::Caps::builder("audio/x-raw") + .field("format", "S16LE") + .field("rate", gst::IntRange::new(8000_i32, 48000)) + .field("channels", 1) + .build(); + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &sink_caps, + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } + + fn change_state( + &self, + element: &Self::Type, + transition: gst::StateChange, + ) -> Result { + gst_info!(CAT, obj: element, "Changing state {:?}", transition); + + let mut success = self.parent_change_state(element, transition)?; + + match transition { + gst::StateChange::PausedToReady => { + self.disconnect(element); + } + gst::StateChange::ReadyToPaused => { + success = gst::StateChangeSuccess::NoPreroll; + } + gst::StateChange::PlayingToPaused => { + success = gst::StateChangeSuccess::NoPreroll; + } + _ => (), + } + + Ok(success) + } + + fn provide_clock(&self, _element: &Self::Type) -> Option { + Some(gst::SystemClock::obtain()) + } +} diff --git a/src/transcriber/mod.rs b/src/transcriber/mod.rs new file mode 100644 index 0000000..9d0477a --- /dev/null +++ b/src/transcriber/mod.rs @@ -0,0 +1,28 @@ +// Copyright (C) 2022 Rafael Caricio +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use glib::prelude::*; +use gst::glib; + +mod imp; + +glib::wrapper! { + pub struct Transcriber(ObjectSubclass) @extends gst::Element, gst::Object; +} + +unsafe impl Send for Transcriber {} +unsafe impl Sync for Transcriber {} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "vosk_transcriber", + gst::Rank::None, + Transcriber::static_type(), + ) +} \ No newline at end of file diff --git a/src/vosk_client/mod.rs b/src/vosk_client/mod.rs new file mode 100644 index 0000000..a4bccca --- /dev/null +++ b/src/vosk_client/mod.rs @@ -0,0 +1,35 @@ +use serde_derive::{Deserialize, Serialize}; + +#[derive(Deserialize, Serialize, Debug)] +pub struct Configuration { + /// Sample rate the audio will be provided at. + sample_rate: i32, + + /// Show time ranges of each word in the transcription. + words: bool, +} + +impl Configuration { + pub fn new(sample_rate: i32) -> Self { + Self { + sample_rate, + // We always want to receive the words with their time ranges. + words: true, + } + } +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct Transcript { + pub result: Vec, + pub text: String, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct WordInfo { + #[serde(rename = "conf")] + pub confidence: f64, + pub word: String, + pub start: f64, + pub end: f64, +} \ No newline at end of file