diff --git a/komorebi-recorder/src/av/gstreamer.rs b/komorebi-recorder/src/av/gstreamer.rs new file mode 100644 index 00000000..c0fe9571 --- /dev/null +++ b/komorebi-recorder/src/av/gstreamer.rs @@ -0,0 +1,459 @@ +//! Decoding recorded video on Linux with GStreamer. +//! +//! GStreamer is the OS media stack on Linux, as AVFoundation is on a Mac and Media Foundation on +//! Windows, so this decodes in process and does not run `ffmpeg`. Each decode is a pipeline, +//! `filesrc ! decodebin ! videoconvert ! videoscale ! ! appsink`: the picture arrives as BGRA +//! at the size wanted (the pipeline converts and scales, so a contact sheet never holds more pixels +//! than it shows). Starting a decode at a time seeks to the key frame before it and decodes forward, +//! and [`super::Selector`] says which of the decoded frames are the answer, as it does for Media +//! Foundation. +//! +//! An [`Asset`] keeps the path and what was read when it was opened (size, duration), not a +//! pipeline: every [`Asset::start`] builds a new one, so a file that changes after it was opened is +//! read as it is then. +//! +//! The pipeline needs a plugin that decodes the file's codec. The Nix package puts H.264 and HEVC +//! decoders (`gst-libav`) on the plugin path; a file whose codec has none says so in the error +//! from [`Asset::open`]. +//! +//! # Tests +//! +//! The rule for which decoded frames answer a request is `Selector`, whose tests run on every +//! host. The tests here make a video with `ffmpeg` and read it back, and stop without one. + +use super::Frame; +use super::Scale; +use super::Selector; +use color_eyre::eyre; +use color_eyre::eyre::WrapErr; +use gstreamer as gst; +use gstreamer::prelude::*; +use gstreamer_app as gst_app; +use gstreamer_video as gst_video; +use std::collections::VecDeque; +use std::path::Path; +use std::path::PathBuf; +use std::sync::OnceLock; + +/// How long to wait for the pipeline to start (the first picture) or for one more frame. +const WAIT: gst::ClockTime = gst::ClockTime::from_seconds(15); + +pub(crate) fn init() -> eyre::Result<()> { + static INIT: OnceLock> = OnceLock::new(); + INIT.get_or_init(|| gst::init().map_err(|error| error.to_string())) + .as_ref() + .map(|_| ()) + .map_err(|why| eyre::Report::msg(format!("GStreamer could not start: {why}"))) +} + +/// The pipeline that decodes `path` to BGRA pictures of `size` (the video's own if `None`). +fn pipeline( + path: &Path, + size: Option<(u32, u32)>, +) -> eyre::Result<(gst::Pipeline, gst_app::AppSink)> { + init()?; + if !path.is_file() { + eyre::bail!("{} does not exist", path.display()); + } + + let caps = match size { + Some((width, height)) => format!("video/x-raw,format=BGRA,width={width},height={height}"), + None => "video/x-raw,format=BGRA".to_string(), + }; + // a bounded queue, so that a decode that is read slowly does not run ahead into memory, and + // no clock, so that it decodes as fast as it is read + let description = format!( + "filesrc name=src ! decodebin ! videoconvert ! videoscale ! {caps} ! \ + appsink name=sink sync=false max-buffers=4 drop=false" + ); + let pipeline = gst::parse::launch(&description) + .wrap_err("could not build the GStreamer pipeline")? + .downcast::() + .map_err(|_| eyre::Report::msg("the GStreamer pipeline is not a pipeline"))?; + pipeline + .by_name("src") + .ok_or_else(|| eyre::Report::msg("the pipeline has no file source"))? + .set_property("location", path.to_string_lossy().as_ref()); + let sink = pipeline + .by_name("sink") + .ok_or_else(|| eyre::Report::msg("the pipeline has no sink"))? + .downcast::() + .map_err(|_| eyre::Report::msg("the pipeline's sink is not an appsink"))?; + Ok((pipeline, sink)) +} + +/// What GStreamer said went wrong, if it said, else `fallback`. +fn failure(pipeline: &gst::Pipeline, fallback: &str) -> eyre::Report { + if let Some(bus) = pipeline.bus() { + while let Some(message) = bus.pop() { + if let gst::MessageView::Error(error) = message.view() { + return eyre::Report::msg(format!( + "{fallback}: {}{}", + error.error(), + error + .debug() + .map(|debug| format!(" ({debug})")) + .unwrap_or_default() + )); + } + } + } + eyre::Report::msg(fallback.to_string()) +} + +/// An opened video file: its path, size and duration. +/// +/// Opening reads those facts and decodes one picture. Each [`Asset::start`] builds a new pipeline +/// on the file and starts where it is told to. +pub struct Asset { + path: PathBuf, + width: u32, + height: u32, + duration_s: f64, +} + +impl Asset { + /// Opens a video file. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - GStreamer cannot start; + /// - `path` is not a file that exists (` does not exist`); + /// - GStreamer cannot decode the file as a video, for example because no plugin decodes its + /// codec (`could not open as a video`, with GStreamer's reason). + pub fn open(path: &Path) -> eyre::Result { + let (pipeline, sink) = pipeline(path, None)?; + let opened = || -> eyre::Result { + pipeline + .set_state(gst::State::Paused) + .wrap_err("GStreamer could not start decoding")?; + let sample = sink.try_pull_preroll(WAIT).ok_or_else(|| { + failure( + &pipeline, + &format!("could not open {} as a video", path.display()), + ) + })?; + let info = sample + .caps() + .ok_or_else(|| eyre::Report::msg("the video has no picture format")) + .and_then(|caps| { + gst_video::VideoInfo::from_caps(caps).wrap_err("the video has no picture size") + })?; + let duration_s = pipeline + .query_duration::() + .map_or(0.0, |duration| duration.nseconds() as f64 / 1e9); + Ok(Self { + path: path.to_path_buf(), + width: info.width(), + height: info.height(), + duration_s, + }) + }; + let asset = opened(); + let _ = pipeline.set_state(gst::State::Null); + asset + } + + /// Returns the size the video was recorded at, as `(width, height)` in pixels. + #[must_use] + pub fn size(&self) -> (u32, u32) { + (self.width, self.height) + } + + /// Returns the duration GStreamer reports for the file, in seconds, or `0.0` if it reports + /// none (some truncated files). + #[must_use] + pub fn duration_s(&self) -> f64 { + self.duration_s + } + + /// Begins decoding at `from_s` seconds into the file, up to `to_s` if given (else the end). + /// + /// The first frame returned is the one on screen at `from_s`, which can be earlier than it. A + /// negative `from_s` is `0`. The end is `to_s`, or the file's + /// [`duration_s`](Asset::duration_s) if it is `None`, and is never before `from_s`: a file + /// whose duration is unknown (`0.0`) therefore decodes little more than the frame on screen at + /// `from_s` unless `to_s` is given. A frame at exactly the end is returned, and none after it. + /// + /// The pipeline gives BGRA frames of the size `scale` asks for (see [`Scale`]). When `from_s` + /// is not `0` it seeks to the key frame at or before it, and the frames from there up to + /// `from_s` are decoded and passed over: a seek costs the frames between a key frame and the + /// time asked for. + /// + /// # Errors + /// + /// Returns an error when: + /// + /// - the file cannot be opened again (see [`Asset::open`]); + /// - the seek to `from_s` fails (`could not seek in the video`). + pub fn start(&self, from_s: f64, to_s: Option, scale: Scale) -> eyre::Result { + let from = from_s.max(0.0); + let end = to_s.unwrap_or(self.duration_s).max(from); + let size = match scale { + Scale::Native => None, + Scale::Width(wide) => { + let wide = wide.clamp(2, self.width.max(2)); + let high = (u64::from(wide) * u64::from(self.height) + / u64::from(self.width.max(1))) + .max(2); + // codecs like even sizes + Some((wide & !1, (high as u32) & !1)) + } + }; + + let (pipeline, sink) = pipeline(&self.path, size)?; + let started = || -> eyre::Result<()> { + pipeline + .set_state(gst::State::Paused) + .wrap_err("GStreamer could not start decoding")?; + // wait until the first picture is ready: only then can the pipeline seek + if sink.try_pull_preroll(WAIT).is_none() { + return Err(failure(&pipeline, "could not start decoding the video")); + } + if from > 0.0 { + // to the key frame at or before it; the frames up to `from` are decoded and + // passed over by the selector + pipeline + .seek_simple( + gst::SeekFlags::FLUSH + | gst::SeekFlags::KEY_UNIT + | gst::SeekFlags::SNAP_BEFORE, + gst::ClockTime::from_nseconds((from * 1e9) as u64), + ) + .wrap_err("could not seek in the video")?; + } + pipeline + .set_state(gst::State::Playing) + .wrap_err("GStreamer could not start decoding")?; + Ok(()) + }; + if let Err(error) = started() { + let _ = pipeline.set_state(gst::State::Null); + return Err(error); + } + + Ok(Reader { + pipeline, + sink, + selector: Selector::new(from, end), + ready: VecDeque::new(), + ended: false, + }) + } +} + +/// A started decode. Dropping it cancels the decode. +/// +/// Made only by [`Asset::start`]. It moves forward only: to look at another time, start another +/// decode. The frames it returns are the ones a `Selector` picks from what the pipeline decodes, +/// so they begin with the frame on screen at the start of the range. +pub struct Reader { + pipeline: gst::Pipeline, + sink: gst_app::AppSink, + selector: Selector, + ready: VecDeque, + ended: bool, +} + +impl Drop for Reader { + fn drop(&mut self) { + let _ = self.pipeline.set_state(gst::State::Null); + } +} + +/// When a decoded frame is shown, in the file's own time: the time zero is its first frame. +/// +/// The buffer's timestamp is in the time of the media, which an MP4's edit list (which `ffmpeg` and +/// the recorder write to hide the frames an encoder delays) puts later than the time the frame is +/// shown at. The stream time, which the sample's segment gives, is the second. +fn shown_at(sample: &gst::Sample, buffer: &gst::BufferRef) -> Option { + let pts = buffer.pts()?; + sample + .segment() + .and_then(|segment| segment.downcast_ref::()) + .and_then(|segment| segment.to_stream_time(pts)) + .or(Some(pts)) +} + +/// One decoded sample as a tightly packed BGRA frame. +fn frame_of(sample: &gst::Sample) -> eyre::Result { + let caps = sample + .caps() + .ok_or_else(|| eyre::Report::msg("a decoded frame has no picture format"))?; + let info = gst_video::VideoInfo::from_caps(caps).wrap_err("a decoded frame has no size")?; + let buffer = sample + .buffer() + .ok_or_else(|| eyre::Report::msg("a decoded frame has no picture"))?; + let map = buffer + .map_readable() + .wrap_err("could not read a decoded frame")?; + + let (width, height) = (info.width() as usize, info.height() as usize); + let stride = usize::try_from(info.stride()[0]).unwrap_or(width * 4); + let data = map.as_slice(); + if data.len() < stride * height.saturating_sub(1) + width * 4 { + eyre::bail!("a decoded frame is smaller than its size says"); + } + + // the rows may be padded; the frame is not + let mut bgra = Vec::with_capacity(width * height * 4); + for row in 0..height { + let start = row * stride; + bgra.extend_from_slice(&data[start..start + width * 4]); + } + // what draws the picture reads the fourth byte as transparency, and a source with no alpha + // may leave it as it likes + for pixel in bgra.as_chunks_mut::<4>().0 { + pixel[3] = 255; + } + + Ok(Frame { + pts_s: shown_at(sample, buffer).map_or(0.0, |time| time.nseconds() as f64 / 1e9), + width: width as u32, + height: height as u32, + bgra, + }) +} + +impl Reader { + /// The next decoded frame of the file, or `None` at its end. + fn decode(&mut self) -> eyre::Result> { + match self.sink.try_pull_sample(WAIT) { + Some(sample) => frame_of(&sample).map(Some), + None if self.sink.is_eos() => Ok(None), + None => Err(failure(&self.pipeline, "could not read a frame")), + } + } + + /// Returns the next frame, or `None` at the end of the range. + /// + /// Frames come in order. Each is BGRA with the alpha byte set to opaque. + /// + /// # Errors + /// + /// Returns an error when the pipeline fails or stops giving frames before the end of the file + /// (`could not read a frame`, with GStreamer's reason), or a decoded frame cannot be read as a + /// picture. + pub fn next_frame(&mut self) -> eyre::Result> { + loop { + if let Some(frame) = self.ready.pop_front() { + return Ok(Some(frame)); + } + if self.selector.done() { + return Ok(None); + } + + let more = if self.ended { + Vec::new() + } else if let Some(frame) = self.decode()? { + self.selector.push(frame) + } else { + self.ended = true; + self.selector.finish() + }; + + if self.ended && more.is_empty() { + return Ok(None); + } + self.ready.extend(more); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testkit::NUMBERED_HEIGHT; + use crate::testkit::NUMBERED_WIDTH; + use crate::testkit::write_numbered_video; + + fn video(name: &str, frames: u32, fps: u32) -> Option { + let dir = + std::env::temp_dir().join(format!("komorebi-recorder-gst-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("a temporary directory"); + let path = dir.join(name); + write_numbered_video(&path, frames, fps).then_some(path) + } + + #[test] + fn a_video_is_opened_and_its_size_and_duration_are_read() { + let Some(path) = video("open.mp4", 30, 30) else { + return; + }; + let asset = Asset::open(&path).expect("open"); + assert_eq!(asset.size(), (NUMBERED_WIDTH, NUMBERED_HEIGHT)); + assert!( + (asset.duration_s() - 1.0).abs() < 0.2, + "duration {}", + asset.duration_s() + ); + } + + #[test] + fn every_frame_is_decoded_in_order_at_the_size_of_the_video() { + let Some(path) = video("all.mp4", 30, 30) else { + return; + }; + let asset = Asset::open(&path).expect("open"); + let mut reader = asset.start(0.0, None, Scale::Native).expect("start"); + let mut times = Vec::new(); + while let Some(frame) = reader.next_frame().expect("a frame") { + assert_eq!( + (frame.width, frame.height), + (NUMBERED_WIDTH, NUMBERED_HEIGHT) + ); + assert_eq!(frame.bgra.len(), (frame.width * frame.height * 4) as usize); + assert!(frame.bgra.chunks(4).all(|pixel| pixel[3] == 255)); + times.push(frame.pts_s); + } + assert_eq!( + times.len(), + 30, + "duration {} frames at {times:?}", + asset.duration_s() + ); + assert!(times.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn a_decode_that_starts_late_begins_with_the_frame_on_screen_then() { + let Some(path) = video("seek.mp4", 90, 30) else { + return; + }; + let asset = Asset::open(&path).expect("open"); + let mut reader = asset.start(1.5, Some(2.0), Scale::Native).expect("start"); + let first = reader.next_frame().expect("a frame").expect("one"); + // the frame on screen at 1.5 s, which is at or just before it + assert!( + first.pts_s <= 1.5 && first.pts_s > 1.5 - 1.0 / 30.0 - 0.001, + "first frame at {}", + first.pts_s + ); + let mut last = first.pts_s; + while let Some(frame) = reader.next_frame().expect("a frame") { + last = frame.pts_s; + } + assert!(last <= 2.0 + 0.001 && last > 1.9, "last frame at {last}"); + } + + #[test] + fn frames_are_scaled_by_the_decoder() { + let Some(path) = video("scale.mp4", 10, 30) else { + return; + }; + let asset = Asset::open(&path).expect("open"); + let mut reader = asset.start(0.0, None, Scale::Width(64)).expect("start"); + let frame = reader.next_frame().expect("a frame").expect("one"); + assert_eq!((frame.width, frame.height), (64, 32)); + } + + #[test] + fn a_file_that_is_not_there_says_so() { + let error = Asset::open(Path::new("/nonexistent/video.mp4")) + .err() + .expect("an error"); + assert!(error.to_string().contains("does not exist"), "{error}"); + } +} diff --git a/komorebi-recorder/src/av/mod.rs b/komorebi-recorder/src/av/mod.rs index 729c38ae..de393be1 100644 --- a/komorebi-recorder/src/av/mod.rs +++ b/komorebi-recorder/src/av/mod.rs @@ -63,28 +63,36 @@ /// Returns `true` if and only if this host can decode the recorded video. /// -/// That is macOS and Windows. Elsewhere [`Asset::open`] always fails. +/// That is macOS, Windows and Linux. Elsewhere [`Asset::open`] always fails. /// /// # Example /// /// ```rust /// assert_eq!( /// komorebi_recorder::av::supported(), -/// cfg!(any(target_os = "macos", windows)) +/// cfg!(any(target_os = "macos", windows, target_os = "linux")) /// ); /// ``` #[must_use] pub const fn supported() -> bool { - cfg!(any(target_os = "macos", windows)) + cfg!(any(target_os = "macos", windows, target_os = "linux")) } +#[cfg(target_os = "linux")] +mod gstreamer; #[cfg(windows)] mod mf; #[cfg(target_os = "macos")] mod reader; -#[cfg(not(any(target_os = "macos", windows)))] +#[cfg(not(any(target_os = "macos", windows, target_os = "linux")))] mod unsupported; +#[cfg(target_os = "linux")] +pub use gstreamer::Asset; +#[cfg(target_os = "linux")] +pub use gstreamer::Reader; +#[cfg(target_os = "linux")] +pub(crate) use gstreamer::init; #[cfg(windows)] pub use mf::Asset; #[cfg(windows)] @@ -93,9 +101,9 @@ pub use mf::Reader; pub use reader::Asset; #[cfg(target_os = "macos")] pub use reader::Reader; -#[cfg(not(any(target_os = "macos", windows)))] +#[cfg(not(any(target_os = "macos", windows, target_os = "linux")))] pub use unsupported::Asset; -#[cfg(not(any(target_os = "macos", windows)))] +#[cfg(not(any(target_os = "macos", windows, target_os = "linux")))] pub use unsupported::Reader; /// The size to decode frames at. @@ -177,7 +185,7 @@ pub struct Frame { /// [`done`]: Selector::done /// [`finish`]: Selector::finish #[derive(Debug)] -#[cfg_attr(not(windows), allow(dead_code))] +#[cfg_attr(not(any(windows, target_os = "linux")), allow(dead_code))] pub(crate) struct Selector { from_s: f64, end_s: f64, @@ -187,7 +195,7 @@ pub(crate) struct Selector { done: bool, } -#[cfg_attr(not(windows), allow(dead_code))] +#[cfg_attr(not(any(windows, target_os = "linux")), allow(dead_code))] impl Selector { /// Creates a selector for the frames shown from `from_s` to `end_s`, both in seconds of the /// file's time.