feat(import): merge re-imports from a linked source instead of duplicating (#618)

crates/yaak-models/src/models.rs

@@ -3022,6 +3022,123 @@ impl<'s> TryFrom<&Row<'s>> for PluginKeyValue {
     }
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default, TS)]
+#[serde(default, rename_all = "camelCase")]
+#[ts(export, export_to = "gen_models.ts")]
+#[enum_def(table_name = "import_sources")]
+pub struct ImportSource {
+    #[ts(type = "\"import_source\"")]
+    pub model: String,
+    pub id: String,
+    pub created_at: NaiveDateTime,
+    pub updated_at: NaiveDateTime,
+    pub workspace_id: String,
+
+    pub importer: String,
+    pub origin: String,
+    pub origin_label: String,
+    pub last_imported_at: NaiveDateTime,
+}
+
+impl UpsertModelInfo for ImportSource {
+    fn table_name() -> impl IntoTableRef + IntoIden {
+        ImportSourceIden::Table
+    }
+
+    fn id_column() -> impl IntoIden + Eq + Clone {
+        ImportSourceIden::Id
+    }
+
+    fn generate_id() -> String {
+        generate_prefixed_id("im")
+    }
+
+    fn order_by() -> (impl IntoColumnRef, Order) {
+        (ImportSourceIden::CreatedAt, Desc)
+    }
+
+    fn get_id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn insert_values(
+        self,
+        source: &UpdateSource,
+    ) -> DbResult<Vec<(impl IntoIden + Eq, impl Into<SimpleExpr>)>> {
+        use ImportSourceIden::*;
+        Ok(vec![
+            (CreatedAt, upsert_date(source, self.created_at)),
+            (UpdatedAt, upsert_date(source, self.updated_at)),
+            (WorkspaceId, self.workspace_id.into()),
+            (Importer, self.importer.into()),
+            (Origin, self.origin.into()),
+            (OriginLabel, self.origin_label.into()),
+            (LastImportedAt, self.last_imported_at.into()),
+        ])
+    }
+
+    fn update_columns() -> Vec<impl IntoIden> {
+        vec![
+            ImportSourceIden::UpdatedAt,
+            ImportSourceIden::Importer,
+            ImportSourceIden::Origin,
+            ImportSourceIden::OriginLabel,
+            ImportSourceIden::LastImportedAt,
+        ]
+    }
+
+    fn from_row(row: &Row) -> rusqlite::Result<Self>
+    where
+        Self: Sized,
+    {
+        Ok(Self {
+            id: row.get("id")?,
+            model: row.get("model")?,
+            created_at: row.get("created_at")?,
+            updated_at: row.get("updated_at")?,
+            workspace_id: row.get("workspace_id")?,
+            importer: row.get("importer")?,
+            origin: row.get("origin")?,
+            origin_label: row.get("origin_label")?,
+            last_imported_at: row.get("last_imported_at")?,
+        })
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default, TS)]
+#[serde(default, rename_all = "camelCase")]
+#[ts(export, export_to = "gen_models.ts")]
+#[enum_def(table_name = "import_source_resources")]
+pub struct ImportSourceResource {
+    #[ts(type = "\"import_source_resource\"")]
+    pub model: String,
+    pub created_at: NaiveDateTime,
+    pub updated_at: NaiveDateTime,
+
+    pub import_source_id: String,
+    pub source_key: String,
+    pub model_type: String,
+    pub model_id: String,
+    pub snapshot: String,
+}
+
+impl<'s> TryFrom<&Row<'s>> for ImportSourceResource {
+    type Error = rusqlite::Error;
+
+    fn try_from(r: &Row<'s>) -> std::result::Result<Self, Self::Error> {
+        Ok(Self {
+            model: r.get("model")?,
+            created_at: r.get("created_at")?,
+            updated_at: r.get("updated_at")?,
+            import_source_id: r.get("import_source_id")?,
+            source_key: r.get("source_key")?,
+            model_type: r.get("model_type")?,
+            model_id: r.get("model_id")?,
+            snapshot: r.get("snapshot")?,
+        })
+    }
+}
+
 /// Only used as a `from_row` fallback for an unparseable settings column. The
 /// value a *new* model gets comes from that model's `Default` impl.
 fn default_request_message_size_setting() -> InheritedIntSetting {
@@ -3093,6 +3210,7 @@ define_any_model! {
     HttpRequest,
     HttpResponse,
     HttpResponseEvent,
+    ImportSource,
     KeyValue,
     Plugin,
     Settings,
@@ -3125,6 +3243,7 @@ impl<'de> Deserialize<'de> for AnyModel {
             Some(m) if m == "http_request" => HttpRequest(fv(value).unwrap()),
             Some(m) if m == "http_response" => HttpResponse(fv(value).unwrap()),
             Some(m) if m == "http_response_event" => HttpResponseEvent(fv(value).unwrap()),
+            Some(m) if m == "import_source" => ImportSource(fv(value).unwrap()),
             Some(m) if m == "key_value" => KeyValue(fv(value).unwrap()),
             Some(m) if m == "plugin" => Plugin(fv(value).unwrap()),
             Some(m) if m == "settings" => Settings(fv(value).unwrap()),

crates/yaak-models/src/queries/import_source_resources.rs

@@ -0,0 +1,94 @@
+use crate::client_db::ClientDb;
+use crate::error::Result;
+use crate::models::{ImportSourceResource, ImportSourceResourceIden};
+use sea_query::ExprTrait;
+use sea_query::Keyword::CurrentTimestamp;
+use sea_query::{Asterisk, Cond, Expr, OnConflict, Query, SqliteQueryBuilder};
+use sea_query_rusqlite::RusqliteBinder;
+
+impl<'a> ClientDb<'a> {
+    pub fn list_import_source_resources(
+        &self,
+        import_source_id: &str,
+    ) -> Result<Vec<ImportSourceResource>> {
+        let (sql, params) = Query::select()
+            .from(ImportSourceResourceIden::Table)
+            .column(Asterisk)
+            .cond_where(Expr::col(ImportSourceResourceIden::ImportSourceId).eq(import_source_id))
+            .build_rusqlite(SqliteQueryBuilder);
+        let mut stmt = self.conn().prepare(sql.as_str())?;
+        let items = stmt.query_map(&*params.as_params(), |row| row.try_into())?;
+        Ok(items.filter_map(|v| v.ok()).collect())
+    }
+
+    pub fn upsert_import_source_resource(
+        &self,
+        resource: &ImportSourceResource,
+    ) -> Result<ImportSourceResource> {
+        let (sql, params) = Query::insert()
+            .into_table(ImportSourceResourceIden::Table)
+            .columns([
+                ImportSourceResourceIden::CreatedAt,
+                ImportSourceResourceIden::UpdatedAt,
+                ImportSourceResourceIden::ImportSourceId,
+                ImportSourceResourceIden::SourceKey,
+                ImportSourceResourceIden::ModelType,
+                ImportSourceResourceIden::ModelId,
+                ImportSourceResourceIden::Snapshot,
+            ])
+            .values_panic([
+                CurrentTimestamp.into(),
+                CurrentTimestamp.into(),
+                resource.import_source_id.as_str().into(),
+                resource.source_key.as_str().into(),
+                resource.model_type.as_str().into(),
+                resource.model_id.as_str().into(),
+                resource.snapshot.as_str().into(),
+            ])
+            .on_conflict(
+                OnConflict::columns([
+                    ImportSourceResourceIden::ImportSourceId,
+                    ImportSourceResourceIden::SourceKey,
+                ])
+                .update_columns([
+                    ImportSourceResourceIden::UpdatedAt,
+                    ImportSourceResourceIden::ModelType,
+                    ImportSourceResourceIden::ModelId,
+                    ImportSourceResourceIden::Snapshot,
+                ])
+                .to_owned(),
+            )
+            .returning_all()
+            .build_rusqlite(SqliteQueryBuilder);
+
+        let mut stmt = self.conn().prepare(sql.as_str())?;
+        let m = stmt.query_row(&*params.as_params(), |row| row.try_into())?;
+        Ok(m)
+    }
+
+    pub fn delete_import_source_resource(
+        &self,
+        import_source_id: &str,
+        source_key: &str,
+    ) -> Result<()> {
+        let (sql, params) = Query::delete()
+            .from_table(ImportSourceResourceIden::Table)
+            .cond_where(
+                Cond::all()
+                    .add(Expr::col(ImportSourceResourceIden::ImportSourceId).eq(import_source_id))
+                    .add(Expr::col(ImportSourceResourceIden::SourceKey).eq(source_key)),
+            )
+            .build_rusqlite(SqliteQueryBuilder);
+        self.conn().execute(sql.as_str(), &*params.as_params())?;
+        Ok(())
+    }
+
+    pub fn delete_import_source_resources(&self, import_source_id: &str) -> Result<()> {
+        let (sql, params) = Query::delete()
+            .from_table(ImportSourceResourceIden::Table)
+            .cond_where(Expr::col(ImportSourceResourceIden::ImportSourceId).eq(import_source_id))
+            .build_rusqlite(SqliteQueryBuilder);
+        self.conn().execute(sql.as_str(), &*params.as_params())?;
+        Ok(())
+    }
+}

crates/yaak-models/src/queries/import_sources.rs

@@ -0,0 +1,45 @@
+use crate::client_db::ClientDb;
+use crate::error::Result;
+use crate::models::{ImportSource, ImportSourceIden};
+use crate::util::UpdateSource;
+
+impl<'a> ClientDb<'a> {
+    pub fn get_import_source(&self, id: &str) -> Result<ImportSource> {
+        self.find_one(ImportSourceIden::Id, id)
+    }
+
+    pub fn list_import_sources(&self, workspace_id: &str) -> Result<Vec<ImportSource>> {
+        self.find_many(ImportSourceIden::WorkspaceId, workspace_id, None)
+    }
+
+    pub fn list_import_sources_by_origin(&self, origin: &str) -> Result<Vec<ImportSource>> {
+        self.find_many(ImportSourceIden::Origin, origin, None)
+    }
+
+    pub fn find_import_source(
+        &self,
+        workspace_id: &str,
+        importer: &str,
+        origin: &str,
+    ) -> Result<Option<ImportSource>> {
+        let sources = self.list_import_sources(workspace_id)?;
+        Ok(sources.into_iter().find(|s| s.importer == importer && s.origin == origin))
+    }
+
+    pub fn upsert_import_source(
+        &self,
+        import_source: &ImportSource,
+        source: &UpdateSource,
+    ) -> Result<ImportSource> {
+        self.upsert(import_source, source)
+    }
+
+    pub fn delete_import_source(
+        &self,
+        import_source: &ImportSource,
+        source: &UpdateSource,
+    ) -> Result<ImportSource> {
+        self.delete_import_source_resources(&import_source.id)?;
+        self.delete(import_source, source)
+    }
+}

crates/yaak-models/src/queries/mod.rs

@@ -11,6 +11,8 @@ mod grpc_requests;
 mod http_requests;
 mod http_response_events;
 mod http_responses;
+mod import_source_resources;
+mod import_sources;
 mod key_values;
 mod model_changes;
 mod plugin_key_values;

crates/yaak-models/src/queries/workspaces.rs

@@ -6,8 +6,9 @@ use crate::models::{
     AnyModel, CookieJar, CookieJarIden, Environment, EnvironmentIden, Folder, FolderIden,
     GraphQlIntrospection, GraphQlIntrospectionIden, GrpcConnection, GrpcConnectionIden, GrpcEvent,
     GrpcEventIden, GrpcRequest, GrpcRequestIden, HttpRequest, HttpRequestHeader, HttpRequestIden,
-    HttpResponse, HttpResponseEvent, HttpResponseEventIden, HttpResponseIden,
-    ResolvedHttpRequestSettings, ResolvedSetting, SyncState, SyncStateIden, WebsocketConnection,
+    HttpResponse, HttpResponseEvent, HttpResponseEventIden, HttpResponseIden, ImportSource,
+    ImportSourceIden, ResolvedHttpRequestSettings, ResolvedSetting, SyncState, SyncStateIden,
+    WebsocketConnection,
     WebsocketConnectionIden, WebsocketEvent, WebsocketEventIden, WebsocketRequest,
     WebsocketRequestIden, Workspace, WorkspaceIden, WorkspaceMeta, WorkspaceMetaIden,
 };
@@ -85,6 +86,10 @@ impl<'a> ClientDb<'a> {
             self.delete_many_untracked::<Folder>(FolderIden::WorkspaceId, wid)?;
             self.delete_many_untracked::<Environment>(EnvironmentIden::WorkspaceId, wid)?;
             self.delete_many_untracked::<CookieJar>(CookieJarIden::WorkspaceId, wid)?;
+            for import_source in self.list_import_sources(wid)? {
+                self.delete_import_source_resources(&import_source.id)?;
+            }
+            self.delete_many_untracked::<ImportSource>(ImportSourceIden::WorkspaceId, wid)?;
             self.delete_many_untracked::<SyncState>(SyncStateIden::WorkspaceId, wid)?;
             self.delete_many_untracked::<WorkspaceMeta>(WorkspaceMetaIden::WorkspaceId, wid)?;
             self.delete(workspace, source)

crates/yaak-models/src/util.rs

@@ -111,6 +111,90 @@ pub struct ImportPlanWarning {
     pub detail: String,
 }
 
+/// Where an import's contents came from, used to link the committed workspace back to it.
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export, export_to = "gen_util.ts")]
+pub struct ImportOrigin {
+    /// The absolute file path or URL the contents were read from.
+    pub origin: String,
+    pub label: String,
+}
+
+/// The model types an import plan can contain.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, TS)]
+#[serde(rename_all = "snake_case")]
+#[ts(export, export_to = "gen_util.ts")]
+pub enum ImportResourceType {
+    Environment,
+    Folder,
+    GrpcRequest,
+    HttpRequest,
+    WebsocketRequest,
+    Workspace,
+}
+
+impl ImportResourceType {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ImportResourceType::Environment => "environment",
+            ImportResourceType::Folder => "folder",
+            ImportResourceType::GrpcRequest => "grpc_request",
+            ImportResourceType::HttpRequest => "http_request",
+            ImportResourceType::WebsocketRequest => "websocket_request",
+            ImportResourceType::Workspace => "workspace",
+        }
+    }
+
+    pub fn from_str(value: &str) -> Option<Self> {
+        match value {
+            "environment" => Some(ImportResourceType::Environment),
+            "folder" => Some(ImportResourceType::Folder),
+            "grpc_request" => Some(ImportResourceType::GrpcRequest),
+            "http_request" => Some(ImportResourceType::HttpRequest),
+            "websocket_request" => Some(ImportResourceType::WebsocketRequest),
+            "workspace" => Some(ImportResourceType::Workspace),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, TS)]
+#[serde(rename_all = "snake_case")]
+#[ts(export, export_to = "gen_util.ts")]
+pub enum ImportPlanAction {
+    Create,
+    Update,
+    Delete,
+    Unchanged,
+    KeepLocal,
+    Conflict,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, TS)]
+#[serde(rename_all = "snake_case")]
+#[ts(export, export_to = "gen_util.ts")]
+pub enum ImportConflictResolution {
+    KeepMine,
+    TakeSource,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, TS)]
+#[serde(rename_all = "camelCase")]
+#[ts(export, export_to = "gen_util.ts")]
+pub struct ImportPlanItem {
+    pub action: ImportPlanAction,
+    pub model: ImportResourceType,
+    pub model_id: String,
+    pub name: String,
+    /// Planned parent folder ID for incoming resources; current parent for deletions.
+    #[ts(optional)]
+    pub parent_id: Option<String>,
+    pub selected: bool,
+    #[ts(optional)]
+    pub resolution: Option<ImportConflictResolution>,
+}
+
 #[derive(Debug, Deserialize, Serialize, TS)]
 #[serde(rename_all = "camelCase")]
 #[ts(export, export_to = "gen_util.ts")]
@@ -120,8 +204,16 @@ pub struct ImportPlan {
     pub resources: BatchUpsertResult,
     pub warnings: Vec<ImportPlanWarning>,
 
-    /// Stable source key for every model in `resources`, keyed by its freshly minted ID.
+    /// Stable source key for every model in `resources`, keyed by its planned ID.
     pub source_keys: BTreeMap<String, String>,
+
+    /// One entry per plannable resource; commit applies only the selected ones.
+    #[serde(default)]
+    pub items: Vec<ImportPlanItem>,
+
+    #[serde(default)]
+    #[ts(optional)]
+    pub origin: Option<ImportOrigin>,
 }
 
 pub fn get_workspace_export_resources(