mirror of
https://github.com/LGUG2Z/komorebi.git
synced 2026-11-17 06:11:21 +01:00
feat(linux): capture the screen with KDE's screencast, read by GStreamer
The displays are the compositor's outputs, with the id komorebi for KDE gives a monitor with no serial number. A screencast of one is a PipeWire node that pipewiresrc reads, stamped on the master clock, into a segment written by the capture writer. KWin offers the screencast only to a program whose desktop file asks for it, and says so when it does not.
This commit is contained in:
1 parent
97aa82751c
commit
a061757598
3 files changed
+411
-3
No files matched your search
@@ -0,0 +1,215 @@
|
||||
//! Capturing the screen on Linux: KDE's screencast of each output, written by GStreamer.
|
||||
//!
|
||||
//! The displays are the compositor's outputs ([`super::wayland::outputs`]). A display's `id` is
|
||||
//! the FNV-1a hash of the output's name, which is how komorebi for KDE makes the id of a monitor
|
||||
//! that reports no serial number (it hashes the make, model and serial when there is one, which a
|
||||
//! Wayland client is not told), so the two agree for most monitors and may not for one with a
|
||||
//! serial number. `frame` is the output's place in the layout in logical pixels, which is where
|
||||
//! komorebi has it, and `scale` is how many pixels the video has for each of them.
|
||||
//!
|
||||
//! What is captured needs the compositor's trust: see [`has_permission`].
|
||||
|
||||
use super::screencast::Source;
|
||||
use super::wayland;
|
||||
use crate::capture::CaptureBackend;
|
||||
use crate::capture::CaptureConfig;
|
||||
use crate::capture::DisplayDescriptor;
|
||||
use crate::capture::RunningCapture;
|
||||
use crate::capture::gst::DisplayCapture;
|
||||
use color_eyre::eyre;
|
||||
use std::path::Path;
|
||||
|
||||
/// FNV-1a, the hash komorebi for KDE makes a monitor's id with.
|
||||
fn stable_id(text: &str) -> u32 {
|
||||
let mut hash: u32 = 0x811c_9dc5;
|
||||
for byte in text.bytes() {
|
||||
hash ^= u32::from(byte);
|
||||
hash = hash.wrapping_mul(0x0100_0193);
|
||||
}
|
||||
hash
|
||||
}
|
||||
|
||||
/// Returns the display that an output is.
|
||||
fn display_of(output: &wayland::Output) -> DisplayDescriptor {
|
||||
let (x, y) = output.position.unwrap_or((0, 0));
|
||||
// an output with no `xdg_output` is laid out by its mode and its integer scale
|
||||
let (width, height) = output.logical_size.unwrap_or((
|
||||
output.mode.0 / output.scale.max(1),
|
||||
output.mode.1 / output.scale.max(1),
|
||||
));
|
||||
let scale = if width > 0 {
|
||||
f64::from(output.mode.0) / f64::from(width)
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
DisplayDescriptor {
|
||||
id: stable_id(&output.label()),
|
||||
frame: [
|
||||
f64::from(x),
|
||||
f64::from(y),
|
||||
f64::from(width),
|
||||
f64::from(height),
|
||||
],
|
||||
width_points: i64::from(width),
|
||||
height_points: i64::from(height),
|
||||
asleep: false,
|
||||
scale,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if and only if this host can capture the screen: it is built for it.
|
||||
#[must_use]
|
||||
pub fn supported() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Returns `true` if and only if the compositor offers this program its screencast.
|
||||
///
|
||||
/// KWin offers `zkde_screencast_unstable_v1` only to a program whose desktop file asks for it
|
||||
/// (`X-KDE-Wayland-Interfaces`), so this is `false` until that file is installed, and on a
|
||||
/// compositor that is not KWin.
|
||||
#[must_use]
|
||||
pub fn has_permission() -> bool {
|
||||
wayland::screencast_offered()
|
||||
}
|
||||
|
||||
/// Makes sure the compositor offers this program its screencast.
|
||||
///
|
||||
/// Nothing is asked of the user: KWin decides by the desktop file, and there is nothing to click.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error that says what is missing when there is no compositor, or it offers no
|
||||
/// screencast to this program.
|
||||
pub fn ensure_permission() -> eyre::Result<()> {
|
||||
wayland::outputs()?;
|
||||
if wayland::screencast_offered() {
|
||||
Ok(())
|
||||
} else {
|
||||
eyre::bail!(
|
||||
"the compositor does not offer a screencast to komorebi-recorder: KWin does that only \
|
||||
for a program whose desktop file lists zkde_screencast_unstable_v1 in \
|
||||
X-KDE-Wayland-Interfaces; `komorebi-recorder doctor` says how to install it"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the displays the compositor has now.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error when there is no compositor to ask.
|
||||
pub fn active_displays() -> eyre::Result<Vec<DisplayDescriptor>> {
|
||||
Ok(wayland::outputs()?.iter().map(display_of).collect())
|
||||
}
|
||||
|
||||
/// Returns the id of the main display (the one at the top left of the layout, as KDE has no other
|
||||
/// notion of a main display here), or `0` if there are none.
|
||||
#[must_use]
|
||||
pub fn main_display_id() -> u32 {
|
||||
main_display().map_or(0, |display| display.id)
|
||||
}
|
||||
|
||||
/// Returns the main display: the one nearest the top left of the layout.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error when there are no displays.
|
||||
pub fn main_display() -> eyre::Result<DisplayDescriptor> {
|
||||
active_displays()?
|
||||
.into_iter()
|
||||
.min_by(|a, b| {
|
||||
(a.frame[0], a.frame[1])
|
||||
.partial_cmp(&(b.frame[0], b.frame[1]))
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
})
|
||||
.ok_or_else(|| eyre::Report::msg("there is no display"))
|
||||
}
|
||||
|
||||
/// Returns the playable duration of a video file as GStreamer reads it, in seconds.
|
||||
#[must_use]
|
||||
pub fn media_duration_s(path: &Path) -> Option<f64> {
|
||||
crate::av::Asset::open(path)
|
||||
.ok()
|
||||
.map(|asset| asset.duration_s())
|
||||
.filter(|duration| *duration > 0.0)
|
||||
}
|
||||
|
||||
/// The Linux [`CaptureBackend`]: outputs from the compositor, and a screencast of each.
|
||||
pub struct Backend;
|
||||
|
||||
impl CaptureBackend for Backend {
|
||||
fn active_displays(&self) -> eyre::Result<Vec<DisplayDescriptor>> {
|
||||
active_displays()
|
||||
}
|
||||
|
||||
fn start(
|
||||
&self,
|
||||
display: &DisplayDescriptor,
|
||||
segment: u32,
|
||||
config: &CaptureConfig,
|
||||
dir: &Path,
|
||||
) -> eyre::Result<Box<dyn RunningCapture>> {
|
||||
// the output is found again by the id it was given, which is the hash of its name
|
||||
let output = wayland::outputs()?
|
||||
.into_iter()
|
||||
.find(|output| stable_id(&output.label()) == display.id)
|
||||
.ok_or_else(|| {
|
||||
eyre::Report::msg(format!(
|
||||
"display {} is not available for capture (yet)",
|
||||
display.id
|
||||
))
|
||||
})?;
|
||||
Ok(Box::new(DisplayCapture::start_with(
|
||||
display,
|
||||
segment,
|
||||
config,
|
||||
dir,
|
||||
|sink| Ok(Box::new(Source::open(output.global, sink)?)),
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn output(name: &str, mode: (i32, i32), scale: i32) -> wayland::Output {
|
||||
wayland::Output {
|
||||
global: 1,
|
||||
name: name.to_string(),
|
||||
mode,
|
||||
scale,
|
||||
..wayland::Output::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_display_has_the_id_komorebi_for_kde_gives_its_monitor() {
|
||||
// the same FNV-1a of the same text as komorebi for KDE's own test of its ids
|
||||
assert_eq!(stable_id(""), 0x811c_9dc5);
|
||||
assert_eq!(stable_id("a"), 0xe40c_292c);
|
||||
assert_eq!(
|
||||
display_of(&output("DP-1", (1920, 1080), 1)).id,
|
||||
stable_id("DP-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_output_with_no_xdg_output_is_laid_out_by_its_scale() {
|
||||
let display = display_of(&output("eDP-1", (2880, 1800), 2));
|
||||
assert_eq!(display.frame, [0.0, 0.0, 1440.0, 900.0]);
|
||||
assert_eq!((display.width_points, display.height_points), (1440, 900));
|
||||
assert!((display.scale - 2.0).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fractional_scale_is_what_the_mode_is_over_the_layout() {
|
||||
let mut wide = output("DP-2", (3840, 2160), 2);
|
||||
wide.position = Some((1536, 0));
|
||||
wide.logical_size = Some((2560, 1440));
|
||||
let display = display_of(&wide);
|
||||
assert_eq!(display.frame, [1536.0, 0.0, 2560.0, 1440.0]);
|
||||
assert!((display.scale - 1.5).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
//! Linux, as KDE Plasma: the host that records komorebi for KDE.
|
||||
//!
|
||||
//! The master clock, the child processes and the shell are Linux's own. The rest of the host
|
||||
//! (`apps`, `capture`, `power`, `session`, `trace`) is still the `unsupported` host's, and is
|
||||
//! The master clock, the child processes, the shell and the screen capture are Linux's own. The
|
||||
//! rest of the host (`apps`, `power`, `session`, `trace`) is still the `unsupported` host's, and is
|
||||
//! replaced one module at a time as each is written (see `docs/PLATFORM.md`).
|
||||
|
||||
// the pieces of the fallback that Linux has its own version of are not used
|
||||
@@ -9,12 +9,14 @@
|
||||
#[path = "../unsupported.rs"]
|
||||
mod fallback;
|
||||
|
||||
pub mod capture;
|
||||
pub mod clock;
|
||||
pub mod process;
|
||||
mod screencast;
|
||||
pub mod shell;
|
||||
pub mod wayland;
|
||||
|
||||
pub use fallback::apps;
|
||||
pub use fallback::capture;
|
||||
pub use fallback::power;
|
||||
pub use fallback::session;
|
||||
pub use fallback::trace;
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
//! The pictures of one output, from the compositor's screencast.
|
||||
//!
|
||||
//! The compositor puts the output's pictures on a PipeWire node ([`wayland::Stream`]), and this
|
||||
//! reads the node with GStreamer's `pipewiresrc`, converts each picture to BGRA, and hands it to a
|
||||
//! [`Sink`]. The pictures are stamped with the master clock: GStreamer's clock is the system's
|
||||
//! monotonic clock, which is the one the recorder's master clock reads (see
|
||||
//! [`super::clock`]), so a picture's running time plus the pipeline's base time is the moment it
|
||||
//! was taken, on the master clock. A picture with no time is stamped when it arrives.
|
||||
//!
|
||||
//! The source ends when it is dropped. If the compositor ends the stream, or the pipeline fails,
|
||||
//! the [`Sink`] is told why, and the segment records it as its error.
|
||||
|
||||
use super::wayland;
|
||||
use crate::capture::gst::Frame;
|
||||
use crate::capture::gst::Sink;
|
||||
use crate::clock;
|
||||
use color_eyre::eyre;
|
||||
use color_eyre::eyre::WrapErr;
|
||||
use gstreamer as gst;
|
||||
use gstreamer::prelude::*;
|
||||
use gstreamer_app as gst_app;
|
||||
use gstreamer_video as gst_video;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// How often the pipeline's bus and the compositor's stream are looked at for an end.
|
||||
const WATCH: gst::ClockTime = gst::ClockTime::from_mseconds(250);
|
||||
|
||||
/// A running screencast of one output: dropping it stops it.
|
||||
pub struct Source {
|
||||
pipeline: gst::Pipeline,
|
||||
stop: Arc<AtomicBool>,
|
||||
watcher: Option<JoinHandle<()>>,
|
||||
// after the pipeline in the order of dropping: the node is read until the pipeline is stopped
|
||||
_stream: Arc<wayland::Stream>,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
/// Starts the screencast of the output with the Wayland global `global`, and gives its
|
||||
/// pictures to `sink`.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error when the compositor does not offer the screencast, or refuses it (see
|
||||
/// [`wayland::Stream::open`]), or when GStreamer cannot read the node (`pipewiresrc` is a
|
||||
/// plugin of PipeWire, and has to be on the plugin path).
|
||||
pub fn open(global: u32, sink: Sink) -> eyre::Result<Self> {
|
||||
crate::av::init()?;
|
||||
let stream = Arc::new(wayland::Stream::open(global)?);
|
||||
|
||||
// the compositor makes pictures when the screen changes, and a picture that has not been
|
||||
// taken by the time the next arrives is not worth keeping: two buffers, the older dropped
|
||||
let description = format!(
|
||||
"pipewiresrc name=src path={} do-timestamp=true always-copy=true ! videoconvert ! \
|
||||
video/x-raw,format=BGRA ! appsink name=sink sync=false max-buffers=2 drop=true",
|
||||
stream.node()
|
||||
);
|
||||
let pipeline = gst::parse::launch(&description)
|
||||
.wrap_err("could not build the GStreamer pipeline that reads the screencast")?
|
||||
.downcast::<gst::Pipeline>()
|
||||
.map_err(|_| eyre::Report::msg("the GStreamer pipeline is not a pipeline"))?;
|
||||
let appsink = pipeline
|
||||
.by_name("sink")
|
||||
.ok_or_else(|| eyre::Report::msg("the pipeline has no sink"))?
|
||||
.downcast::<gst_app::AppSink>()
|
||||
.map_err(|_| eyre::Report::msg("the pipeline's sink is not an appsink"))?;
|
||||
|
||||
let clock_of = pipeline.downgrade();
|
||||
let frames = sink.clone();
|
||||
appsink.set_callbacks(
|
||||
gst_app::AppSinkCallbacks::builder()
|
||||
.new_sample(move |appsink| {
|
||||
let sample = appsink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
|
||||
let base = clock_of.upgrade().and_then(|pipeline| pipeline.base_time());
|
||||
match frame_of(&sample, base) {
|
||||
Ok(frame) => frames.frame(frame),
|
||||
Err(error) => frames.fail(format!("{error:#}")),
|
||||
}
|
||||
Ok(gst::FlowSuccess::Ok)
|
||||
})
|
||||
.build(),
|
||||
);
|
||||
|
||||
pipeline.set_state(gst::State::Playing).wrap_err(
|
||||
"GStreamer could not read the screencast (is the PipeWire plugin on the plugin path?)",
|
||||
)?;
|
||||
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let watcher = {
|
||||
let stop = stop.clone();
|
||||
let pipeline = pipeline.clone();
|
||||
let stream = stream.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("screencast".to_string())
|
||||
.spawn(move || watch(&pipeline, &stream, &stop, &sink))?
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
pipeline,
|
||||
stop,
|
||||
watcher: Some(watcher),
|
||||
_stream: stream,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Source {
|
||||
fn drop(&mut self) {
|
||||
self.stop.store(true, Ordering::Relaxed);
|
||||
let _ = self.pipeline.set_state(gst::State::Null);
|
||||
if let Some(watcher) = self.watcher.take() {
|
||||
let _ = watcher.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tells `sink` when the compositor ends the stream, or the pipeline fails.
|
||||
fn watch(pipeline: &gst::Pipeline, stream: &wayland::Stream, stop: &AtomicBool, sink: &Sink) {
|
||||
let Some(bus) = pipeline.bus() else {
|
||||
return;
|
||||
};
|
||||
while !stop.load(Ordering::Relaxed) {
|
||||
if stream.is_closed() {
|
||||
sink.fail("the compositor ended the screencast (the output went away?)".to_string());
|
||||
return;
|
||||
}
|
||||
let message =
|
||||
bus.timed_pop_filtered(WATCH, &[gst::MessageType::Error, gst::MessageType::Eos]);
|
||||
match message.as_deref().map(gst::MessageRef::view) {
|
||||
Some(gst::MessageView::Error(error)) => {
|
||||
sink.fail(format!("the screencast failed: {}", error.error()));
|
||||
return;
|
||||
}
|
||||
Some(gst::MessageView::Eos(_)) => {
|
||||
sink.fail("the screencast ended".to_string());
|
||||
return;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One picture from the pipeline as a [`Frame`], stamped on the master clock.
|
||||
fn frame_of(sample: &gst::Sample, base: Option<gst::ClockTime>) -> eyre::Result<Frame> {
|
||||
let caps = sample
|
||||
.caps()
|
||||
.ok_or_else(|| eyre::Report::msg("a picture of the screencast has no format"))?;
|
||||
let info = gst_video::VideoInfo::from_caps(caps).wrap_err("a picture has no size")?;
|
||||
let buffer = sample
|
||||
.buffer_owned()
|
||||
.ok_or_else(|| eyre::Report::msg("a picture of the screencast has no pixels"))?;
|
||||
let (width, height) = (info.width(), info.height());
|
||||
|
||||
// the rows of a picture may be padded, and the segment's frames are not
|
||||
let stride = usize::try_from(info.stride()[0]).unwrap_or(0);
|
||||
let row = width as usize * 4;
|
||||
let buffer = if stride == row {
|
||||
buffer
|
||||
} else {
|
||||
let map = buffer
|
||||
.map_readable()
|
||||
.wrap_err("could not read a picture of the screencast")?;
|
||||
let mut packed = Vec::with_capacity(row * height as usize);
|
||||
for line in 0..height as usize {
|
||||
let start = line * stride;
|
||||
let end = start + row;
|
||||
if end > map.len() {
|
||||
eyre::bail!("a picture of the screencast is smaller than its size says");
|
||||
}
|
||||
packed.extend_from_slice(&map[start..end]);
|
||||
}
|
||||
gst::Buffer::from_slice(packed)
|
||||
};
|
||||
|
||||
let arrived = clock::mono_now_ns();
|
||||
let pts_ns = match (buffer.pts(), base) {
|
||||
// the running time of a picture, and the moment the pipeline started, on the clock that
|
||||
// the master clock reads
|
||||
(Some(pts), Some(base)) => (pts + base).nseconds(),
|
||||
_ => arrived,
|
||||
};
|
||||
Ok(Frame {
|
||||
buffer,
|
||||
width,
|
||||
height,
|
||||
// a picture cannot be from the future
|
||||
pts_ns: pts_ns.min(arrived),
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user