From 2ca51125a43ab3786bbe73684bce07d3e9cd14fc Mon Sep 17 00:00:00 2001 From: Gregory Schier Date: Sun, 1 Mar 2026 16:30:43 -0800 Subject: [PATCH] Improve plugin source modeling and runtime dedup (#414) --- AGENTS.md | 1 + crates-tauri/yaak-app/src/lib.rs | 10 +- crates-tauri/yaak-app/src/models_ext.rs | 26 +++-- crates/yaak-models/bindings/gen_models.ts | 4 +- ...000_plugin-source-and-unique-directory.sql | 33 ++++++ crates/yaak-models/src/models.rs | 43 ++++++++ crates/yaak-models/src/queries/plugins.rs | 6 +- crates/yaak-plugins/src/api.rs | 4 +- crates/yaak-plugins/src/install.rs | 3 +- crates/yaak-plugins/src/manager.rs | 101 +++++++++++++++--- .../template-function-timestamp/package.json | 2 + .../components/Settings/SettingsPlugins.tsx | 15 +-- src-web/components/core/Dropdown.tsx | 6 +- 13 files changed, 210 insertions(+), 44 deletions(-) create mode 100644 crates/yaak-models/migrations/20260301000000_plugin-source-and-unique-directory.sql diff --git a/AGENTS.md b/AGENTS.md index 209538bd..6d9a7968 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1 +1,2 @@ - Tag safety: app releases use `v*` tags and CLI releases use `yaak-cli-*` tags; always confirm which one is requested before retagging. +- Do not commit, push, or tag without explicit approval diff --git a/crates-tauri/yaak-app/src/lib.rs b/crates-tauri/yaak-app/src/lib.rs index 48f6877e..b9298342 100644 --- a/crates-tauri/yaak-app/src/lib.rs +++ b/crates-tauri/yaak-app/src/lib.rs @@ -38,7 +38,7 @@ use yaak_mac_window::AppHandleMacWindowExt; use yaak_models::models::{ AnyModel, CookieJar, Environment, GrpcConnection, GrpcConnectionState, GrpcEvent, GrpcEventType, HttpRequest, HttpResponse, HttpResponseEvent, HttpResponseState, Plugin, - Workspace, WorkspaceMeta, + PluginSource, Workspace, WorkspaceMeta, }; use yaak_models::util::{BatchUpsertResult, UpdateSource, get_workspace_export_resources}; use yaak_plugins::events::{ @@ -1354,7 +1354,13 @@ async fn cmd_install_plugin( window: WebviewWindow, ) -> YaakResult { let plugin = app_handle.db().upsert_plugin( - &Plugin { directory: directory.into(), url, enabled: true, ..Default::default() }, + &Plugin { + directory: directory.into(), + url, + enabled: true, + source: PluginSource::Filesystem, + ..Default::default() + }, &UpdateSource::from_window_label(window.label()), )?; diff --git a/crates-tauri/yaak-app/src/models_ext.rs b/crates-tauri/yaak-app/src/models_ext.rs index 90833087..14b5eebf 100644 --- a/crates-tauri/yaak-app/src/models_ext.rs +++ b/crates-tauri/yaak-app/src/models_ext.rs @@ -15,6 +15,7 @@ use yaak_models::error::Result; use yaak_models::models::{AnyModel, GraphQlIntrospection, GrpcEvent, Settings, WebsocketEvent}; use yaak_models::query_manager::QueryManager; use yaak_models::util::UpdateSource; +use yaak_plugins::manager::PluginManager; const MODEL_CHANGES_RETENTION_HOURS: i64 = 1; const MODEL_CHANGES_POLL_INTERVAL_MS: u64 = 1000; @@ -255,23 +256,32 @@ pub(crate) fn models_upsert_graphql_introspection( } #[tauri::command] -pub(crate) fn models_workspace_models( +pub(crate) async fn models_workspace_models( window: WebviewWindow, workspace_id: Option<&str>, + plugin_manager: State<'_, PluginManager>, ) -> Result { - let db = window.db(); let mut l: Vec = Vec::new(); - // Add the settings - l.push(db.get_settings().into()); + // Add the global models + { + let db = window.db(); + l.push(db.get_settings().into()); + l.append(&mut db.list_workspaces()?.into_iter().map(Into::into).collect()); + l.append(&mut db.list_key_values()?.into_iter().map(Into::into).collect()); + } - // Add global models - l.append(&mut db.list_workspaces()?.into_iter().map(Into::into).collect()); - l.append(&mut db.list_key_values()?.into_iter().map(Into::into).collect()); - l.append(&mut db.list_plugins()?.into_iter().map(Into::into).collect()); + let plugins = { + let db = window.db(); + db.list_plugins()? + }; + + let plugins = plugin_manager.resolve_plugins_for_runtime_from_db(plugins).await; + l.append(&mut plugins.into_iter().map(Into::into).collect()); // Add the workspace children if let Some(wid) = workspace_id { + let db = window.db(); l.append(&mut db.list_cookie_jars(wid)?.into_iter().map(Into::into).collect()); l.append(&mut db.list_environments_ensure_base(wid)?.into_iter().map(Into::into).collect()); l.append(&mut db.list_folders(wid)?.into_iter().map(Into::into).collect()); diff --git a/crates/yaak-models/bindings/gen_models.ts b/crates/yaak-models/bindings/gen_models.ts index da1da73c..da7cf754 100644 --- a/crates/yaak-models/bindings/gen_models.ts +++ b/crates/yaak-models/bindings/gen_models.ts @@ -67,7 +67,9 @@ export type ParentAuthentication = { authentication: Record, authen export type ParentHeaders = { headers: Array, }; -export type Plugin = { model: "plugin", id: string, createdAt: string, updatedAt: string, checkedAt: string | null, directory: string, enabled: boolean, url: string | null, }; +export type Plugin = { model: "plugin", id: string, createdAt: string, updatedAt: string, checkedAt: string | null, directory: string, enabled: boolean, url: string | null, source: PluginSource, }; + +export type PluginSource = "bundled" | "filesystem" | "registry"; export type PluginKeyValue = { model: "plugin_key_value", createdAt: string, updatedAt: string, pluginName: string, key: string, value: string, }; diff --git a/crates/yaak-models/migrations/20260301000000_plugin-source-and-unique-directory.sql b/crates/yaak-models/migrations/20260301000000_plugin-source-and-unique-directory.sql new file mode 100644 index 00000000..dca3a159 --- /dev/null +++ b/crates/yaak-models/migrations/20260301000000_plugin-source-and-unique-directory.sql @@ -0,0 +1,33 @@ +ALTER TABLE plugins + ADD COLUMN source TEXT DEFAULT 'filesystem' NOT NULL; + +-- Existing registry installs have a URL; classify them first. +UPDATE plugins +SET source = 'registry' +WHERE url IS NOT NULL; + +-- Best-effort bundled backfill for legacy rows. +UPDATE plugins +SET source = 'bundled' +WHERE source = 'filesystem' + AND ( + -- Normalize separators so this also works for Windows paths. + replace(directory, '\', '/') LIKE '%/vendored/plugins/%' + OR replace(directory, '\', '/') LIKE '%/vendored-plugins/%' + ); + +-- Keep one row per exact directory before adding uniqueness. +-- Tie-break by recency. +WITH ranked AS (SELECT id, + ROW_NUMBER() OVER ( + PARTITION BY directory + ORDER BY updated_at DESC, + created_at DESC + ) AS row_num + FROM plugins) +DELETE +FROM plugins +WHERE id IN (SELECT id FROM ranked WHERE row_num > 1); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_plugins_directory_unique + ON plugins (directory); diff --git a/crates/yaak-models/src/models.rs b/crates/yaak-models/src/models.rs index 95796031..119c775b 100644 --- a/crates/yaak-models/src/models.rs +++ b/crates/yaak-models/src/models.rs @@ -2074,6 +2074,46 @@ pub struct Plugin { pub directory: String, pub enabled: bool, pub url: Option, + pub source: PluginSource, +} + +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[serde(rename_all = "snake_case")] +#[ts(export, export_to = "gen_models.ts")] +pub enum PluginSource { + Bundled, + Filesystem, + Registry, +} + +impl FromStr for PluginSource { + type Err = crate::error::Error; + + fn from_str(s: &str) -> Result { + match s { + "bundled" => Ok(Self::Bundled), + "filesystem" => Ok(Self::Filesystem), + "registry" => Ok(Self::Registry), + _ => Ok(Self::default()), + } + } +} + +impl Display for PluginSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let str = match self { + PluginSource::Bundled => "bundled".to_string(), + PluginSource::Filesystem => "filesystem".to_string(), + PluginSource::Registry => "registry".to_string(), + }; + write!(f, "{}", str) + } +} + +impl Default for PluginSource { + fn default() -> Self { + Self::Filesystem + } } impl UpsertModelInfo for Plugin { @@ -2109,6 +2149,7 @@ impl UpsertModelInfo for Plugin { (Directory, self.directory.into()), (Url, self.url.into()), (Enabled, self.enabled.into()), + (Source, self.source.to_string().into()), ]) } @@ -2119,6 +2160,7 @@ impl UpsertModelInfo for Plugin { PluginIden::Directory, PluginIden::Url, PluginIden::Enabled, + PluginIden::Source, ] } @@ -2135,6 +2177,7 @@ impl UpsertModelInfo for Plugin { url: row.get("url")?, directory: row.get("directory")?, enabled: row.get("enabled")?, + source: PluginSource::from_str(row.get::<_, String>("source")?.as_str()).unwrap(), }) } } diff --git a/crates/yaak-models/src/queries/plugins.rs b/crates/yaak-models/src/queries/plugins.rs index 21451b12..c3b97e14 100644 --- a/crates/yaak-models/src/queries/plugins.rs +++ b/crates/yaak-models/src/queries/plugins.rs @@ -26,6 +26,10 @@ impl<'a> DbContext<'a> { } pub fn upsert_plugin(&self, plugin: &Plugin, source: &UpdateSource) -> Result { - self.upsert(plugin, source) + let mut plugin_to_upsert = plugin.clone(); + if let Some(existing) = self.get_plugin_by_directory(&plugin.directory) { + plugin_to_upsert.id = existing.id; + } + self.upsert(&plugin_to_upsert, source) } } diff --git a/crates/yaak-plugins/src/api.rs b/crates/yaak-plugins/src/api.rs index ad16dad9..f09f92c1 100644 --- a/crates/yaak-plugins/src/api.rs +++ b/crates/yaak-plugins/src/api.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use std::path::Path; use std::str::FromStr; use ts_rs::TS; -use yaak_models::models::Plugin; +use yaak_models::models::{Plugin, PluginSource}; /// Get plugin info from the registry. pub async fn get_plugin( @@ -58,7 +58,7 @@ pub async fn check_plugin_updates( ) -> Result { let name_versions: Vec = plugins .into_iter() - .filter(|p| p.url.is_some()) // Only check plugins with URLs (from registry) + .filter(|p| matches!(p.source, PluginSource::Registry)) // Only check registry-installed plugins .filter_map(|p| match get_plugin_meta(&Path::new(&p.directory)) { Ok(m) => Some(PluginNameVersion { name: m.name, version: m.version }), Err(e) => { diff --git a/crates/yaak-plugins/src/install.rs b/crates/yaak-plugins/src/install.rs index 99acc996..bafd00ab 100644 --- a/crates/yaak-plugins/src/install.rs +++ b/crates/yaak-plugins/src/install.rs @@ -9,7 +9,7 @@ use log::info; use std::fs::{create_dir_all, remove_dir_all}; use std::io::Cursor; use std::sync::Arc; -use yaak_models::models::Plugin; +use yaak_models::models::{Plugin, PluginSource}; use yaak_models::query_manager::QueryManager; use yaak_models::util::UpdateSource; @@ -78,6 +78,7 @@ pub async fn download_and_install( directory: plugin_dir_str.clone(), enabled: true, url: Some(plugin_version.url.clone()), + source: PluginSource::Registry, ..Default::default() }, &UpdateSource::Background, diff --git a/crates/yaak-plugins/src/manager.rs b/crates/yaak-plugins/src/manager.rs index d17eefe1..26400e63 100644 --- a/crates/yaak-plugins/src/manager.rs +++ b/crates/yaak-plugins/src/manager.rs @@ -21,9 +21,10 @@ use crate::events::{ use crate::native_template_functions::{template_function_keyring, template_function_secure}; use crate::nodejs::start_nodejs_plugin_runtime; use crate::plugin_handle::PluginHandle; +use crate::plugin_meta::get_plugin_meta; use crate::server_ws::PluginRuntimeServerWebsocket; use log::{error, info, warn}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::env; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -33,7 +34,7 @@ use tokio::net::TcpListener; use tokio::sync::mpsc::error::TrySendError; use tokio::sync::{Mutex, mpsc, oneshot}; use tokio::time::{Instant, timeout}; -use yaak_models::models::Plugin; +use yaak_models::models::{Plugin, PluginSource}; use yaak_models::query_manager::QueryManager; use yaak_models::util::{UpdateSource, generate_id}; use yaak_templates::error::Error::RenderError; @@ -162,13 +163,14 @@ impl PluginManager { let bundled_dirs = plugin_manager.list_bundled_plugin_dirs().await?; let db = query_manager.connect(); - for dir in bundled_dirs { - if db.get_plugin_by_directory(&dir).is_none() { + for dir in &bundled_dirs { + if db.get_plugin_by_directory(dir).is_none() { db.upsert_plugin( &Plugin { - directory: dir, + directory: dir.clone(), enabled: true, url: None, + source: PluginSource::Bundled, ..Default::default() }, &UpdateSource::Background, @@ -213,6 +215,57 @@ impl PluginManager { read_plugins_dir(&plugins_dir).await } + pub async fn resolve_plugins_for_runtime_from_db(&self, plugins: Vec) -> Vec { + let bundled_dirs = match self.list_bundled_plugin_dirs().await { + Ok(dirs) => dirs, + Err(err) => { + warn!("Failed to read bundled plugin dirs for resolution: {err:?}"); + Vec::new() + } + }; + self.resolve_plugins_for_runtime(plugins, bundled_dirs) + } + + /// Resolve the plugin set for the current runtime instance. + /// + /// Rules: + /// - Drop bundled rows that are not present in this instance's bundled directory list. + /// - Deduplicate by plugin metadata name (fallback to directory key when metadata is unreadable). + /// - Prefer sources in this order: filesystem > registry > bundled. + /// - For same-source conflicts, prefer the most recently installed row (`created_at`). + fn resolve_plugins_for_runtime( + &self, + plugins: Vec, + bundled_dirs: Vec, + ) -> Vec { + let bundled_dir_set: HashSet = bundled_dirs.into_iter().collect(); + let mut selected: HashMap = HashMap::new(); + + for plugin in plugins { + if matches!(plugin.source, PluginSource::Bundled) + && !bundled_dir_set.contains(&plugin.directory) + { + continue; + } + + let key = match get_plugin_meta(Path::new(&plugin.directory)) { + Ok(meta) => meta.name, + Err(_) => format!("__dir__{}", plugin.directory), + }; + + match selected.get(&key) { + Some(existing) if !prefer_plugin(&plugin, existing) => {} + _ => { + selected.insert(key, plugin); + } + } + } + + let mut resolved = selected.into_values().collect::>(); + resolved.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + resolved + } + pub async fn uninstall(&self, plugin_context: &PluginContext, dir: &str) -> Result<()> { let plugin = self.get_plugin_by_dir(dir).await.ok_or(PluginNotFoundErr(dir.to_string()))?; self.remove_plugin(plugin_context, &plugin).await @@ -287,7 +340,8 @@ impl PluginManager { Ok(()) } - /// Initialize all plugins from the provided list. + /// Initialize all plugins from the provided DB list. + /// Plugin candidates are resolved for this runtime instance before initialization. /// Returns a list of (plugin_directory, error_message) for any plugins that failed to initialize. pub async fn initialize_all_plugins( &self, @@ -297,15 +351,18 @@ impl PluginManager { info!("Initializing all plugins"); let start = Instant::now(); let mut errors = Vec::new(); + let plugins = self.resolve_plugins_for_runtime_from_db(plugins).await; + + // Rebuild runtime handles from scratch to avoid stale/duplicate handles. + let existing_handles = { self.plugin_handles.lock().await.clone() }; + for plugin_handle in existing_handles { + if let Err(e) = self.remove_plugin(plugin_context, &plugin_handle).await { + error!("Failed to remove plugin {} {e:?}", plugin_handle.dir); + errors.push((plugin_handle.dir.clone(), e.to_string())); + } + } for plugin in plugins { - // First remove the plugin if it exists and is enabled - if let Some(plugin_handle) = self.get_plugin_by_dir(&plugin.directory).await { - if let Err(e) = self.remove_plugin(plugin_context, &plugin_handle).await { - error!("Failed to remove plugin {} {e:?}", plugin.directory); - continue; - } - } if let Err(e) = self.add_plugin(plugin_context, &plugin).await { warn!("Failed to add plugin {} {e:?}", plugin.directory); errors.push((plugin.directory.clone(), e.to_string())); @@ -1063,6 +1120,24 @@ impl PluginManager { } } +fn source_priority(source: &PluginSource) -> i32 { + match source { + PluginSource::Filesystem => 3, + PluginSource::Registry => 2, + PluginSource::Bundled => 1, + } +} + +fn prefer_plugin(candidate: &Plugin, existing: &Plugin) -> bool { + let candidate_priority = source_priority(&candidate.source); + let existing_priority = source_priority(&existing.source); + if candidate_priority != existing_priority { + return candidate_priority > existing_priority; + } + + candidate.created_at > existing.created_at +} + async fn read_plugins_dir(dir: &PathBuf) -> Result> { let mut result = read_dir(dir).await?; let mut dirs: Vec = vec![]; diff --git a/plugins/template-function-timestamp/package.json b/plugins/template-function-timestamp/package.json index bafcd111..5b7ac0cf 100755 --- a/plugins/template-function-timestamp/package.json +++ b/plugins/template-function-timestamp/package.json @@ -1,5 +1,7 @@ { "name": "@yaak/template-function-timestamp", + "displayName": "Timestamp Template Functions", + "description": "Template functions for dealing with timestamps", "private": true, "version": "0.1.0", "scripts": { diff --git a/src-web/components/Settings/SettingsPlugins.tsx b/src-web/components/Settings/SettingsPlugins.tsx index 300a70de..7298fc7b 100644 --- a/src-web/components/Settings/SettingsPlugins.tsx +++ b/src-web/components/Settings/SettingsPlugins.tsx @@ -15,7 +15,6 @@ import { useDebouncedValue } from '../../hooks/useDebouncedValue'; import { useInstallPlugin } from '../../hooks/useInstallPlugin'; import { usePluginInfo } from '../../hooks/usePluginInfo'; import { usePluginsKey, useRefreshPlugins } from '../../hooks/usePlugins'; -import { appInfo } from '../../lib/appInfo'; import { showConfirmDelete } from '../../lib/confirm'; import { minPromiseMillis } from '../../lib/minPromiseMillis'; import { Button } from '../core/Button'; @@ -33,16 +32,6 @@ import { TabContent, Tabs } from '../core/Tabs/Tabs'; import { EmptyStateText } from '../EmptyStateText'; import { SelectFile } from '../SelectFile'; -function isPluginBundled(plugin: Plugin, vendoredPluginDir: string): boolean { - const normalizedDir = plugin.directory.replace(/\\/g, '/'); - const normalizedVendoredDir = vendoredPluginDir.replace(/\\/g, '/'); - return ( - normalizedDir.includes(normalizedVendoredDir) || - normalizedDir.includes('vendored/plugins') || - normalizedDir.includes('/plugins/') - ); -} - interface SettingsPluginsProps { defaultSubtab?: string; } @@ -50,8 +39,8 @@ interface SettingsPluginsProps { export function SettingsPlugins({ defaultSubtab }: SettingsPluginsProps) { const [directory, setDirectory] = useState(null); const plugins = useAtomValue(pluginsAtom); - const bundledPlugins = plugins.filter((p) => isPluginBundled(p, appInfo.vendoredPluginDir)); - const installedPlugins = plugins.filter((p) => !isPluginBundled(p, appInfo.vendoredPluginDir)); + const bundledPlugins = plugins.filter((p) => p.source === 'bundled'); + const installedPlugins = plugins.filter((p) => p.source !== 'bundled'); const createPlugin = useInstallPlugin(); const refreshPlugins = useRefreshPlugins(); return ( diff --git a/src-web/components/core/Dropdown.tsx b/src-web/components/core/Dropdown.tsx index 6de614fb..22b9f074 100644 --- a/src-web/components/core/Dropdown.tsx +++ b/src-web/components/core/Dropdown.tsx @@ -896,9 +896,9 @@ function MenuItem({ }; const rightSlot = item.submenu ? ( - + ) : ( - (item.rightSlot ?? ) + (item.rightSlot ?? ) ); return ( @@ -937,7 +937,7 @@ function MenuItem({ )} {...props} > -
{item.label}
+
{item.label}
); }