feat(import): merge re-imports from a linked source instead of duplicating (#618)

This commit is contained in:
Gregory Schier
2026-09-01 11:12:07 -07:00
committed by GitHub
parent d461c982ec
commit d2d4b80a09
28 changed files with 2504 additions and 275 deletions
+24
View File
@@ -12,6 +12,7 @@ export type AnyModel =
| HttpRequest
| HttpResponse
| HttpResponseEvent
| ImportSource
| KeyValue
| Plugin
| Settings
@@ -336,6 +337,29 @@ export type HttpUrlParameter = {
export type HttpVersion = "auto" | "http1" | "http2";
export type ImportSource = {
model: "import_source";
id: string;
createdAt: string;
updatedAt: string;
workspaceId: string;
importer: string;
origin: string;
originLabel: string;
lastImportedAt: string;
};
export type ImportSourceResource = {
model: "import_source_resource";
createdAt: string;
updatedAt: string;
importSourceId: string;
sourceKey: string;
modelType: string;
modelId: string;
snapshot: string;
};
export type InheritedBoolSetting = { enabled?: boolean; value: boolean };
export type InheritedHttpVersionSetting = { enabled?: boolean; value: HttpVersion };
+30 -2
View File
@@ -3,6 +3,8 @@ import type { Environment, Folder, GrpcRequest, HttpRequest, WebsocketRequest, W
export type BatchUpsertResult = { workspaces: Array<Workspace>, environments: Array<Environment>, folders: Array<Folder>, httpRequests: Array<HttpRequest>, grpcRequests: Array<GrpcRequest>, websocketRequests: Array<WebsocketRequest>, };
export type ImportConflictResolution = "keep_mine" | "take_source";
/**
* Where a staged import will be committed.
*
@@ -11,10 +13,36 @@ export type BatchUpsertResult = { workspaces: Array<Workspace>, environments: Ar
*/
export type ImportDestination = { "type": "new_workspace" } | { "type": "existing_workspace", workspaceId: string, folderId?: string, };
/**
* Where an import's contents came from, used to link the committed workspace back to it.
*/
export type ImportOrigin = {
/**
* The absolute file path or URL the contents were read from.
*/
origin: string, label: string, };
export type ImportPlan = { importer: string, destination: ImportDestination, resources: BatchUpsertResult, warnings: Array<ImportPlanWarning>,
/**
* Stable source key for every model in `resources`, keyed by its freshly minted ID.
* Stable source key for every model in `resources`, keyed by its planned ID.
*/
sourceKeys: { [key in string]?: string }, };
sourceKeys: { [key in string]?: string },
/**
* One entry per plannable resource; commit applies only the selected ones.
*/
items: Array<ImportPlanItem>, origin?: ImportOrigin, };
export type ImportPlanAction = "create" | "update" | "delete" | "unchanged" | "keep_local" | "conflict";
export type ImportPlanItem = { action: ImportPlanAction, model: ImportResourceType, modelId: string, name: string,
/**
* Planned parent folder ID for incoming resources; current parent for deletions.
*/
parentId?: string, selected: boolean, resolution?: ImportConflictResolution, };
export type ImportPlanWarning = { title: string, detail: string, };
/**
* The model types an import plan can contain.
*/
export type ImportResourceType = "environment" | "folder" | "grpc_request" | "http_request" | "websocket_request" | "workspace";
+1
View File
@@ -12,6 +12,7 @@ export function newStoreData(): ModelStoreData {
http_request: {},
http_response: {},
http_response_event: {},
import_source: {},
key_value: {},
plugin: {},
settings: {},
@@ -0,0 +1,25 @@
CREATE TABLE import_sources
(
id TEXT NOT NULL PRIMARY KEY,
model TEXT DEFAULT 'import_source' NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
workspace_id TEXT NOT NULL,
importer TEXT NOT NULL,
origin TEXT NOT NULL,
origin_label TEXT NOT NULL,
last_imported_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL
);
CREATE TABLE import_source_resources
(
model TEXT DEFAULT 'import_source_resource' NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
import_source_id TEXT NOT NULL,
source_key TEXT NOT NULL,
model_type TEXT NOT NULL,
model_id TEXT NOT NULL,
snapshot TEXT NOT NULL,
PRIMARY KEY (import_source_id, source_key)
);
+119
View File
@@ -3022,6 +3022,123 @@ impl<'s> TryFrom<&Row<'s>> for PluginKeyValue {
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default, TS)]
#[serde(default, rename_all = "camelCase")]
#[ts(export, export_to = "gen_models.ts")]
#[enum_def(table_name = "import_sources")]
pub struct ImportSource {
#[ts(type = "\"import_source\"")]
pub model: String,
pub id: String,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
pub workspace_id: String,
pub importer: String,
pub origin: String,
pub origin_label: String,
pub last_imported_at: NaiveDateTime,
}
impl UpsertModelInfo for ImportSource {
fn table_name() -> impl IntoTableRef + IntoIden {
ImportSourceIden::Table
}
fn id_column() -> impl IntoIden + Eq + Clone {
ImportSourceIden::Id
}
fn generate_id() -> String {
generate_prefixed_id("im")
}
fn order_by() -> (impl IntoColumnRef, Order) {
(ImportSourceIden::CreatedAt, Desc)
}
fn get_id(&self) -> String {
self.id.clone()
}
fn insert_values(
self,
source: &UpdateSource,
) -> DbResult<Vec<(impl IntoIden + Eq, impl Into<SimpleExpr>)>> {
use ImportSourceIden::*;
Ok(vec![
(CreatedAt, upsert_date(source, self.created_at)),
(UpdatedAt, upsert_date(source, self.updated_at)),
(WorkspaceId, self.workspace_id.into()),
(Importer, self.importer.into()),
(Origin, self.origin.into()),
(OriginLabel, self.origin_label.into()),
(LastImportedAt, self.last_imported_at.into()),
])
}
fn update_columns() -> Vec<impl IntoIden> {
vec![
ImportSourceIden::UpdatedAt,
ImportSourceIden::Importer,
ImportSourceIden::Origin,
ImportSourceIden::OriginLabel,
ImportSourceIden::LastImportedAt,
]
}
fn from_row(row: &Row) -> rusqlite::Result<Self>
where
Self: Sized,
{
Ok(Self {
id: row.get("id")?,
model: row.get("model")?,
created_at: row.get("created_at")?,
updated_at: row.get("updated_at")?,
workspace_id: row.get("workspace_id")?,
importer: row.get("importer")?,
origin: row.get("origin")?,
origin_label: row.get("origin_label")?,
last_imported_at: row.get("last_imported_at")?,
})
}
}
#[derive(Debug, Clone, Serialize, Deserialize, Default, TS)]
#[serde(default, rename_all = "camelCase")]
#[ts(export, export_to = "gen_models.ts")]
#[enum_def(table_name = "import_source_resources")]
pub struct ImportSourceResource {
#[ts(type = "\"import_source_resource\"")]
pub model: String,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
pub import_source_id: String,
pub source_key: String,
pub model_type: String,
pub model_id: String,
pub snapshot: String,
}
impl<'s> TryFrom<&Row<'s>> for ImportSourceResource {
type Error = rusqlite::Error;
fn try_from(r: &Row<'s>) -> std::result::Result<Self, Self::Error> {
Ok(Self {
model: r.get("model")?,
created_at: r.get("created_at")?,
updated_at: r.get("updated_at")?,
import_source_id: r.get("import_source_id")?,
source_key: r.get("source_key")?,
model_type: r.get("model_type")?,
model_id: r.get("model_id")?,
snapshot: r.get("snapshot")?,
})
}
}
/// Only used as a `from_row` fallback for an unparseable settings column. The
/// value a *new* model gets comes from that model's `Default` impl.
fn default_request_message_size_setting() -> InheritedIntSetting {
@@ -3093,6 +3210,7 @@ define_any_model! {
HttpRequest,
HttpResponse,
HttpResponseEvent,
ImportSource,
KeyValue,
Plugin,
Settings,
@@ -3125,6 +3243,7 @@ impl<'de> Deserialize<'de> for AnyModel {
Some(m) if m == "http_request" => HttpRequest(fv(value).unwrap()),
Some(m) if m == "http_response" => HttpResponse(fv(value).unwrap()),
Some(m) if m == "http_response_event" => HttpResponseEvent(fv(value).unwrap()),
Some(m) if m == "import_source" => ImportSource(fv(value).unwrap()),
Some(m) if m == "key_value" => KeyValue(fv(value).unwrap()),
Some(m) if m == "plugin" => Plugin(fv(value).unwrap()),
Some(m) if m == "settings" => Settings(fv(value).unwrap()),
@@ -0,0 +1,94 @@
use crate::client_db::ClientDb;
use crate::error::Result;
use crate::models::{ImportSourceResource, ImportSourceResourceIden};
use sea_query::ExprTrait;
use sea_query::Keyword::CurrentTimestamp;
use sea_query::{Asterisk, Cond, Expr, OnConflict, Query, SqliteQueryBuilder};
use sea_query_rusqlite::RusqliteBinder;
impl<'a> ClientDb<'a> {
pub fn list_import_source_resources(
&self,
import_source_id: &str,
) -> Result<Vec<ImportSourceResource>> {
let (sql, params) = Query::select()
.from(ImportSourceResourceIden::Table)
.column(Asterisk)
.cond_where(Expr::col(ImportSourceResourceIden::ImportSourceId).eq(import_source_id))
.build_rusqlite(SqliteQueryBuilder);
let mut stmt = self.conn().prepare(sql.as_str())?;
let items = stmt.query_map(&*params.as_params(), |row| row.try_into())?;
Ok(items.filter_map(|v| v.ok()).collect())
}
pub fn upsert_import_source_resource(
&self,
resource: &ImportSourceResource,
) -> Result<ImportSourceResource> {
let (sql, params) = Query::insert()
.into_table(ImportSourceResourceIden::Table)
.columns([
ImportSourceResourceIden::CreatedAt,
ImportSourceResourceIden::UpdatedAt,
ImportSourceResourceIden::ImportSourceId,
ImportSourceResourceIden::SourceKey,
ImportSourceResourceIden::ModelType,
ImportSourceResourceIden::ModelId,
ImportSourceResourceIden::Snapshot,
])
.values_panic([
CurrentTimestamp.into(),
CurrentTimestamp.into(),
resource.import_source_id.as_str().into(),
resource.source_key.as_str().into(),
resource.model_type.as_str().into(),
resource.model_id.as_str().into(),
resource.snapshot.as_str().into(),
])
.on_conflict(
OnConflict::columns([
ImportSourceResourceIden::ImportSourceId,
ImportSourceResourceIden::SourceKey,
])
.update_columns([
ImportSourceResourceIden::UpdatedAt,
ImportSourceResourceIden::ModelType,
ImportSourceResourceIden::ModelId,
ImportSourceResourceIden::Snapshot,
])
.to_owned(),
)
.returning_all()
.build_rusqlite(SqliteQueryBuilder);
let mut stmt = self.conn().prepare(sql.as_str())?;
let m = stmt.query_row(&*params.as_params(), |row| row.try_into())?;
Ok(m)
}
pub fn delete_import_source_resource(
&self,
import_source_id: &str,
source_key: &str,
) -> Result<()> {
let (sql, params) = Query::delete()
.from_table(ImportSourceResourceIden::Table)
.cond_where(
Cond::all()
.add(Expr::col(ImportSourceResourceIden::ImportSourceId).eq(import_source_id))
.add(Expr::col(ImportSourceResourceIden::SourceKey).eq(source_key)),
)
.build_rusqlite(SqliteQueryBuilder);
self.conn().execute(sql.as_str(), &*params.as_params())?;
Ok(())
}
pub fn delete_import_source_resources(&self, import_source_id: &str) -> Result<()> {
let (sql, params) = Query::delete()
.from_table(ImportSourceResourceIden::Table)
.cond_where(Expr::col(ImportSourceResourceIden::ImportSourceId).eq(import_source_id))
.build_rusqlite(SqliteQueryBuilder);
self.conn().execute(sql.as_str(), &*params.as_params())?;
Ok(())
}
}
@@ -0,0 +1,45 @@
use crate::client_db::ClientDb;
use crate::error::Result;
use crate::models::{ImportSource, ImportSourceIden};
use crate::util::UpdateSource;
impl<'a> ClientDb<'a> {
pub fn get_import_source(&self, id: &str) -> Result<ImportSource> {
self.find_one(ImportSourceIden::Id, id)
}
pub fn list_import_sources(&self, workspace_id: &str) -> Result<Vec<ImportSource>> {
self.find_many(ImportSourceIden::WorkspaceId, workspace_id, None)
}
pub fn list_import_sources_by_origin(&self, origin: &str) -> Result<Vec<ImportSource>> {
self.find_many(ImportSourceIden::Origin, origin, None)
}
pub fn find_import_source(
&self,
workspace_id: &str,
importer: &str,
origin: &str,
) -> Result<Option<ImportSource>> {
let sources = self.list_import_sources(workspace_id)?;
Ok(sources.into_iter().find(|s| s.importer == importer && s.origin == origin))
}
pub fn upsert_import_source(
&self,
import_source: &ImportSource,
source: &UpdateSource,
) -> Result<ImportSource> {
self.upsert(import_source, source)
}
pub fn delete_import_source(
&self,
import_source: &ImportSource,
source: &UpdateSource,
) -> Result<ImportSource> {
self.delete_import_source_resources(&import_source.id)?;
self.delete(import_source, source)
}
}
+2
View File
@@ -11,6 +11,8 @@ mod grpc_requests;
mod http_requests;
mod http_response_events;
mod http_responses;
mod import_source_resources;
mod import_sources;
mod key_values;
mod model_changes;
mod plugin_key_values;
+7 -2
View File
@@ -6,8 +6,9 @@ use crate::models::{
AnyModel, CookieJar, CookieJarIden, Environment, EnvironmentIden, Folder, FolderIden,
GraphQlIntrospection, GraphQlIntrospectionIden, GrpcConnection, GrpcConnectionIden, GrpcEvent,
GrpcEventIden, GrpcRequest, GrpcRequestIden, HttpRequest, HttpRequestHeader, HttpRequestIden,
HttpResponse, HttpResponseEvent, HttpResponseEventIden, HttpResponseIden,
ResolvedHttpRequestSettings, ResolvedSetting, SyncState, SyncStateIden, WebsocketConnection,
HttpResponse, HttpResponseEvent, HttpResponseEventIden, HttpResponseIden, ImportSource,
ImportSourceIden, ResolvedHttpRequestSettings, ResolvedSetting, SyncState, SyncStateIden,
WebsocketConnection,
WebsocketConnectionIden, WebsocketEvent, WebsocketEventIden, WebsocketRequest,
WebsocketRequestIden, Workspace, WorkspaceIden, WorkspaceMeta, WorkspaceMetaIden,
};
@@ -85,6 +86,10 @@ impl<'a> ClientDb<'a> {
self.delete_many_untracked::<Folder>(FolderIden::WorkspaceId, wid)?;
self.delete_many_untracked::<Environment>(EnvironmentIden::WorkspaceId, wid)?;
self.delete_many_untracked::<CookieJar>(CookieJarIden::WorkspaceId, wid)?;
for import_source in self.list_import_sources(wid)? {
self.delete_import_source_resources(&import_source.id)?;
}
self.delete_many_untracked::<ImportSource>(ImportSourceIden::WorkspaceId, wid)?;
self.delete_many_untracked::<SyncState>(SyncStateIden::WorkspaceId, wid)?;
self.delete_many_untracked::<WorkspaceMeta>(WorkspaceMetaIden::WorkspaceId, wid)?;
self.delete(workspace, source)
+93 -1
View File
@@ -111,6 +111,90 @@ pub struct ImportPlanWarning {
pub detail: String,
}
/// Where an import's contents came from, used to link the committed workspace back to it.
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_util.ts")]
pub struct ImportOrigin {
/// The absolute file path or URL the contents were read from.
pub origin: String,
pub label: String,
}
/// The model types an import plan can contain.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, TS)]
#[serde(rename_all = "snake_case")]
#[ts(export, export_to = "gen_util.ts")]
pub enum ImportResourceType {
Environment,
Folder,
GrpcRequest,
HttpRequest,
WebsocketRequest,
Workspace,
}
impl ImportResourceType {
pub fn as_str(&self) -> &'static str {
match self {
ImportResourceType::Environment => "environment",
ImportResourceType::Folder => "folder",
ImportResourceType::GrpcRequest => "grpc_request",
ImportResourceType::HttpRequest => "http_request",
ImportResourceType::WebsocketRequest => "websocket_request",
ImportResourceType::Workspace => "workspace",
}
}
pub fn from_str(value: &str) -> Option<Self> {
match value {
"environment" => Some(ImportResourceType::Environment),
"folder" => Some(ImportResourceType::Folder),
"grpc_request" => Some(ImportResourceType::GrpcRequest),
"http_request" => Some(ImportResourceType::HttpRequest),
"websocket_request" => Some(ImportResourceType::WebsocketRequest),
"workspace" => Some(ImportResourceType::Workspace),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, TS)]
#[serde(rename_all = "snake_case")]
#[ts(export, export_to = "gen_util.ts")]
pub enum ImportPlanAction {
Create,
Update,
Delete,
Unchanged,
KeepLocal,
Conflict,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, TS)]
#[serde(rename_all = "snake_case")]
#[ts(export, export_to = "gen_util.ts")]
pub enum ImportConflictResolution {
KeepMine,
TakeSource,
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_util.ts")]
pub struct ImportPlanItem {
pub action: ImportPlanAction,
pub model: ImportResourceType,
pub model_id: String,
pub name: String,
/// Planned parent folder ID for incoming resources; current parent for deletions.
#[ts(optional)]
pub parent_id: Option<String>,
pub selected: bool,
#[ts(optional)]
pub resolution: Option<ImportConflictResolution>,
}
#[derive(Debug, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export, export_to = "gen_util.ts")]
@@ -120,8 +204,16 @@ pub struct ImportPlan {
pub resources: BatchUpsertResult,
pub warnings: Vec<ImportPlanWarning>,
/// Stable source key for every model in `resources`, keyed by its freshly minted ID.
/// Stable source key for every model in `resources`, keyed by its planned ID.
pub source_keys: BTreeMap<String, String>,
/// One entry per plannable resource; commit applies only the selected ones.
#[serde(default)]
pub items: Vec<ImportPlanItem>,
#[serde(default)]
#[ts(optional)]
pub origin: Option<ImportOrigin>,
}
pub fn get_workspace_export_resources(