feat(linux): capture the screen with KDE's screencast, read by GStreamer

The displays are the compositor's outputs, with the id komorebi for KDE gives a monitor with no
serial number. A screencast of one is a PipeWire node that pipewiresrc reads, stamped on the
master clock, into a segment written by the capture writer. KWin offers the screencast only to
a program whose desktop file asks for it, and says so when it does not.
This commit is contained in:
LGUG2Z committed 2026-10-28 22:50:40 -07:00
1 parent 97aa82751c
commit a061757598
3 files changed
+411 -3

No files matched your search

@@ -0,0 +1,215 @@
//! Capturing the screen on Linux: KDE's screencast of each output, written by GStreamer.
//!
//! The displays are the compositor's outputs ([`super::wayland::outputs`]). A display's `id` is
//! the FNV-1a hash of the output's name, which is how komorebi for KDE makes the id of a monitor
//! that reports no serial number (it hashes the make, model and serial when there is one, which a
//! Wayland client is not told), so the two agree for most monitors and may not for one with a
//! serial number. `frame` is the output's place in the layout in logical pixels, which is where
//! komorebi has it, and `scale` is how many pixels the video has for each of them.
//!
//! What is captured needs the compositor's trust: see [`has_permission`].
use super::screencast::Source;
use super::wayland;
use crate::capture::CaptureBackend;
use crate::capture::CaptureConfig;
use crate::capture::DisplayDescriptor;
use crate::capture::RunningCapture;
use crate::capture::gst::DisplayCapture;
use color_eyre::eyre;
use std::path::Path;
/// FNV-1a, the hash komorebi for KDE makes a monitor's id with.
fn stable_id(text: &str) -> u32 {
let mut hash: u32 = 0x811c_9dc5;
for byte in text.bytes() {
hash ^= u32::from(byte);
hash = hash.wrapping_mul(0x0100_0193);
}
hash
}
/// Returns the display that an output is.
fn display_of(output: &wayland::Output) -> DisplayDescriptor {
let (x, y) = output.position.unwrap_or((0, 0));
// an output with no `xdg_output` is laid out by its mode and its integer scale
let (width, height) = output.logical_size.unwrap_or((
output.mode.0 / output.scale.max(1),
output.mode.1 / output.scale.max(1),
));
let scale = if width > 0 {
f64::from(output.mode.0) / f64::from(width)
} else {
1.0
};
DisplayDescriptor {
id: stable_id(&output.label()),
frame: [
f64::from(x),
f64::from(y),
f64::from(width),
f64::from(height),
],
width_points: i64::from(width),
height_points: i64::from(height),
asleep: false,
scale,
}
}
/// Returns `true` if and only if this host can capture the screen: it is built for it.
#[must_use]
pub fn supported() -> bool {
true
}
/// Returns `true` if and only if the compositor offers this program its screencast.
///
/// KWin offers `zkde_screencast_unstable_v1` only to a program whose desktop file asks for it
/// (`X-KDE-Wayland-Interfaces`), so this is `false` until that file is installed, and on a
/// compositor that is not KWin.
#[must_use]
pub fn has_permission() -> bool {
wayland::screencast_offered()
}
/// Makes sure the compositor offers this program its screencast.
///
/// Nothing is asked of the user: KWin decides by the desktop file, and there is nothing to click.
///
/// # Errors
///
/// Returns an error that says what is missing when there is no compositor, or it offers no
/// screencast to this program.
pub fn ensure_permission() -> eyre::Result<()> {
wayland::outputs()?;
if wayland::screencast_offered() {
Ok(())
} else {
eyre::bail!(
"the compositor does not offer a screencast to komorebi-recorder: KWin does that only \
for a program whose desktop file lists zkde_screencast_unstable_v1 in \
X-KDE-Wayland-Interfaces; `komorebi-recorder doctor` says how to install it"
)
}
}
/// Returns the displays the compositor has now.
///
/// # Errors
///
/// Returns an error when there is no compositor to ask.
pub fn active_displays() -> eyre::Result<Vec<DisplayDescriptor>> {
Ok(wayland::outputs()?.iter().map(display_of).collect())
}
/// Returns the id of the main display (the one at the top left of the layout, as KDE has no other
/// notion of a main display here), or `0` if there are none.
#[must_use]
pub fn main_display_id() -> u32 {
main_display().map_or(0, |display| display.id)
}
/// Returns the main display: the one nearest the top left of the layout.
///
/// # Errors
///
/// Returns an error when there are no displays.
pub fn main_display() -> eyre::Result<DisplayDescriptor> {
active_displays()?
.into_iter()
.min_by(|a, b| {
(a.frame[0], a.frame[1])
.partial_cmp(&(b.frame[0], b.frame[1]))
.unwrap_or(std::cmp::Ordering::Equal)
})
.ok_or_else(|| eyre::Report::msg("there is no display"))
}
/// Returns the playable duration of a video file as GStreamer reads it, in seconds.
#[must_use]
pub fn media_duration_s(path: &Path) -> Option<f64> {
crate::av::Asset::open(path)
.ok()
.map(|asset| asset.duration_s())
.filter(|duration| *duration > 0.0)
}
/// The Linux [`CaptureBackend`]: outputs from the compositor, and a screencast of each.
pub struct Backend;
impl CaptureBackend for Backend {
fn active_displays(&self) -> eyre::Result<Vec<DisplayDescriptor>> {
active_displays()
}
fn start(
&self,
display: &DisplayDescriptor,
segment: u32,
config: &CaptureConfig,
dir: &Path,
) -> eyre::Result<Box<dyn RunningCapture>> {
// the output is found again by the id it was given, which is the hash of its name
let output = wayland::outputs()?
.into_iter()
.find(|output| stable_id(&output.label()) == display.id)
.ok_or_else(|| {
eyre::Report::msg(format!(
"display {} is not available for capture (yet)",
display.id
))
})?;
Ok(Box::new(DisplayCapture::start_with(
display,
segment,
config,
dir,
|sink| Ok(Box::new(Source::open(output.global, sink)?)),
)?))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn output(name: &str, mode: (i32, i32), scale: i32) -> wayland::Output {
wayland::Output {
global: 1,
name: name.to_string(),
mode,
scale,
..wayland::Output::default()
}
}
#[test]
fn a_display_has_the_id_komorebi_for_kde_gives_its_monitor() {
// the same FNV-1a of the same text as komorebi for KDE's own test of its ids
assert_eq!(stable_id(""), 0x811c_9dc5);
assert_eq!(stable_id("a"), 0xe40c_292c);
assert_eq!(
display_of(&output("DP-1", (1920, 1080), 1)).id,
stable_id("DP-1")
);
}
#[test]
fn an_output_with_no_xdg_output_is_laid_out_by_its_scale() {
let display = display_of(&output("eDP-1", (2880, 1800), 2));
assert_eq!(display.frame, [0.0, 0.0, 1440.0, 900.0]);
assert_eq!((display.width_points, display.height_points), (1440, 900));
assert!((display.scale - 2.0).abs() < 1e-9);
}
#[test]
fn a_fractional_scale_is_what_the_mode_is_over_the_layout() {
let mut wide = output("DP-2", (3840, 2160), 2);
wide.position = Some((1536, 0));
wide.logical_size = Some((2560, 1440));
let display = display_of(&wide);
assert_eq!(display.frame, [1536.0, 0.0, 2560.0, 1440.0]);
assert!((display.scale - 1.5).abs() < 1e-9);
}
}
+5 -3
View File
@@ -1,7 +1,7 @@
//! Linux, as KDE Plasma: the host that records komorebi for KDE.
//!
//! The master clock, the child processes and the shell are Linux's own. The rest of the host
//! (`apps`, `capture`, `power`, `session`, `trace`) is still the `unsupported` host's, and is
//! The master clock, the child processes, the shell and the screen capture are Linux's own. The
//! rest of the host (`apps`, `power`, `session`, `trace`) is still the `unsupported` host's, and is
//! replaced one module at a time as each is written (see `docs/PLATFORM.md`).
// the pieces of the fallback that Linux has its own version of are not used
@@ -9,12 +9,14 @@
#[path = "../unsupported.rs"]
mod fallback;
pub mod capture;
pub mod clock;
pub mod process;
mod screencast;
pub mod shell;
pub mod wayland;
pub use fallback::apps;
pub use fallback::capture;
pub use fallback::power;
pub use fallback::session;
pub use fallback::trace;
@@ -0,0 +1,191 @@
//! The pictures of one output, from the compositor's screencast.
//!
//! The compositor puts the output's pictures on a PipeWire node ([`wayland::Stream`]), and this
//! reads the node with GStreamer's `pipewiresrc`, converts each picture to BGRA, and hands it to a
//! [`Sink`]. The pictures are stamped with the master clock: GStreamer's clock is the system's
//! monotonic clock, which is the one the recorder's master clock reads (see
//! [`super::clock`]), so a picture's running time plus the pipeline's base time is the moment it
//! was taken, on the master clock. A picture with no time is stamped when it arrives.
//!
//! The source ends when it is dropped. If the compositor ends the stream, or the pipeline fails,
//! the [`Sink`] is told why, and the segment records it as its error.
use super::wayland;
use crate::capture::gst::Frame;
use crate::capture::gst::Sink;
use crate::clock;
use color_eyre::eyre;
use color_eyre::eyre::WrapErr;
use gstreamer as gst;
use gstreamer::prelude::*;
use gstreamer_app as gst_app;
use gstreamer_video as gst_video;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::thread::JoinHandle;
/// How often the pipeline's bus and the compositor's stream are looked at for an end.
const WATCH: gst::ClockTime = gst::ClockTime::from_mseconds(250);
/// A running screencast of one output: dropping it stops it.
pub struct Source {
pipeline: gst::Pipeline,
stop: Arc<AtomicBool>,
watcher: Option<JoinHandle<()>>,
// after the pipeline in the order of dropping: the node is read until the pipeline is stopped
_stream: Arc<wayland::Stream>,
}
impl Source {
/// Starts the screencast of the output with the Wayland global `global`, and gives its
/// pictures to `sink`.
///
/// # Errors
///
/// Returns an error when the compositor does not offer the screencast, or refuses it (see
/// [`wayland::Stream::open`]), or when GStreamer cannot read the node (`pipewiresrc` is a
/// plugin of PipeWire, and has to be on the plugin path).
pub fn open(global: u32, sink: Sink) -> eyre::Result<Self> {
crate::av::init()?;
let stream = Arc::new(wayland::Stream::open(global)?);
// the compositor makes pictures when the screen changes, and a picture that has not been
// taken by the time the next arrives is not worth keeping: two buffers, the older dropped
let description = format!(
"pipewiresrc name=src path={} do-timestamp=true always-copy=true ! videoconvert ! \
video/x-raw,format=BGRA ! appsink name=sink sync=false max-buffers=2 drop=true",
stream.node()
);
let pipeline = gst::parse::launch(&description)
.wrap_err("could not build the GStreamer pipeline that reads the screencast")?
.downcast::<gst::Pipeline>()
.map_err(|_| eyre::Report::msg("the GStreamer pipeline is not a pipeline"))?;
let appsink = pipeline
.by_name("sink")
.ok_or_else(|| eyre::Report::msg("the pipeline has no sink"))?
.downcast::<gst_app::AppSink>()
.map_err(|_| eyre::Report::msg("the pipeline's sink is not an appsink"))?;
let clock_of = pipeline.downgrade();
let frames = sink.clone();
appsink.set_callbacks(
gst_app::AppSinkCallbacks::builder()
.new_sample(move |appsink| {
let sample = appsink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
let base = clock_of.upgrade().and_then(|pipeline| pipeline.base_time());
match frame_of(&sample, base) {
Ok(frame) => frames.frame(frame),
Err(error) => frames.fail(format!("{error:#}")),
}
Ok(gst::FlowSuccess::Ok)
})
.build(),
);
pipeline.set_state(gst::State::Playing).wrap_err(
"GStreamer could not read the screencast (is the PipeWire plugin on the plugin path?)",
)?;
let stop = Arc::new(AtomicBool::new(false));
let watcher = {
let stop = stop.clone();
let pipeline = pipeline.clone();
let stream = stream.clone();
std::thread::Builder::new()
.name("screencast".to_string())
.spawn(move || watch(&pipeline, &stream, &stop, &sink))?
};
Ok(Self {
pipeline,
stop,
watcher: Some(watcher),
_stream: stream,
})
}
}
impl Drop for Source {
fn drop(&mut self) {
self.stop.store(true, Ordering::Relaxed);
let _ = self.pipeline.set_state(gst::State::Null);
if let Some(watcher) = self.watcher.take() {
let _ = watcher.join();
}
}
}
/// Tells `sink` when the compositor ends the stream, or the pipeline fails.
fn watch(pipeline: &gst::Pipeline, stream: &wayland::Stream, stop: &AtomicBool, sink: &Sink) {
let Some(bus) = pipeline.bus() else {
return;
};
while !stop.load(Ordering::Relaxed) {
if stream.is_closed() {
sink.fail("the compositor ended the screencast (the output went away?)".to_string());
return;
}
let message =
bus.timed_pop_filtered(WATCH, &[gst::MessageType::Error, gst::MessageType::Eos]);
match message.as_deref().map(gst::MessageRef::view) {
Some(gst::MessageView::Error(error)) => {
sink.fail(format!("the screencast failed: {}", error.error()));
return;
}
Some(gst::MessageView::Eos(_)) => {
sink.fail("the screencast ended".to_string());
return;
}
_ => {}
}
}
}
/// One picture from the pipeline as a [`Frame`], stamped on the master clock.
fn frame_of(sample: &gst::Sample, base: Option<gst::ClockTime>) -> eyre::Result<Frame> {
let caps = sample
.caps()
.ok_or_else(|| eyre::Report::msg("a picture of the screencast has no format"))?;
let info = gst_video::VideoInfo::from_caps(caps).wrap_err("a picture has no size")?;
let buffer = sample
.buffer_owned()
.ok_or_else(|| eyre::Report::msg("a picture of the screencast has no pixels"))?;
let (width, height) = (info.width(), info.height());
// the rows of a picture may be padded, and the segment's frames are not
let stride = usize::try_from(info.stride()[0]).unwrap_or(0);
let row = width as usize * 4;
let buffer = if stride == row {
buffer
} else {
let map = buffer
.map_readable()
.wrap_err("could not read a picture of the screencast")?;
let mut packed = Vec::with_capacity(row * height as usize);
for line in 0..height as usize {
let start = line * stride;
let end = start + row;
if end > map.len() {
eyre::bail!("a picture of the screencast is smaller than its size says");
}
packed.extend_from_slice(&map[start..end]);
}
gst::Buffer::from_slice(packed)
};
let arrived = clock::mono_now_ns();
let pts_ns = match (buffer.pts(), base) {
// the running time of a picture, and the moment the pipeline started, on the clock that
// the master clock reads
(Some(pts), Some(base)) => (pts + base).nseconds(),
_ => arrived,
};
Ok(Frame {
buffer,
width,
height,
// a picture cannot be from the future
pts_ns: pts_ns.min(arrived),
})
}