starred/yaak-mountain-loop
1
0
Fork 0
mirror of https://github.com/mountain-loop/yaak.git synced 2026-03-31 14:33:18 +02:00
Code Issues 8 Packages Projects Releases 10 Wiki Activity

Ability to sync environments to folder (#207)

Browse Source
This commit is contained in:
Gregory Schier
2025-05-08 14:10:07 -07:00
committed by GitHub
parent 77cdea2f9f
commit 94d4227bc1
54 changed files with 710 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src-tauri/migrations/20250507140702_remove-ev-sync-states.sql Normal file
View File

@@ -0,0 +1,11 @@
-- There used to be sync code that skipped over environments because we didn't
-- want to sync potentially insecure data. With encryption, it is now possible
-- to sync environments securely. However, there were already sync states in the
-- DB that marked environments as "Synced". Running the sync code on these envs
-- would mark them as deleted by FS (exist in SyncState but not on FS).
--
-- To undo this mess, we have this migration to delete all environment-related
-- sync states so we can sync from a clean slate.
DELETE
FROM sync_states
WHERE model_id LIKE 'ev_%';

20
src-tauri/migrations/20250508161145_public-environments.sql Normal file
View File

@@ -0,0 +1,20 @@
-- Add a public column to represent whether an environment can be shared or exported
ALTER TABLE environments
ADD COLUMN public BOOLEAN DEFAULT FALSE;
-- Add a base column to represent whether an environment is a base or sub environment. We used to
-- do this with environment_id, but we need a more flexible solution now that envs can be optionally
-- synced. E.g., it's now possible to only import a sub environment from a different client without
-- its base environment "parent."
ALTER TABLE environments
ADD COLUMN base BOOLEAN DEFAULT FALSE;
-- SQLite doesn't support dynamic default values, so we update `base` based on the value of
-- environment_id.
UPDATE environments
SET base = TRUE
WHERE environment_id IS NULL;
-- Finally, we drop the old `environment_id` column that will no longer be used
ALTER TABLE environments
DROP COLUMN environment_id;

5
src-tauri/src/lib.rs
View File

@@ -831,7 +831,6 @@ async fn cmd_import_data<R: Runtime>(
.map(|mut v| {
v.id = maybe_gen_id::<Environment>(v.id.as_str(), &mut id_map);
v.workspace_id = maybe_gen_id::<Workspace>(v.workspace_id.as_str(), &mut id_map);
v.environment_id = maybe_gen_id_opt::<Environment>(v.environment_id, &mut id_map);
v
})
.collect();
@@ -985,10 +984,10 @@ async fn cmd_export_data<R: Runtime>(
app_handle: AppHandle<R>,
export_path: &str,
workspace_ids: Vec<&str>,
include_environments: bool,
include_private_environments: bool,
) -> YaakResult<()> {
let export_data =
get_workspace_export_resources(&app_handle, workspace_ids, include_environments).await?;
get_workspace_export_resources(&app_handle, workspace_ids, include_private_environments)?;
let f = File::options()
.create(true)
.truncate(true)

2
src-tauri/vendored/plugins/importer-insomnia/build/index.js generated
View File

@@ -7372,7 +7372,7 @@ function importEnvironment(e, workspaceId) {
createdAt: e.created ? new Date(e.created).toISOString().replace("Z", "") : void 0,
updatedAt: e.updated ? new Date(e.updated).toISOString().replace("Z", "") : void 0,
workspaceId: convertId(workspaceId),
environmentId: e.parentId === workspaceId ? null : convertId(e.parentId),
base: e.parentId === workspaceId ? true : false,
model: "environment",
name: e.name,
variables: Object.entries(e.data).map(([name, value]) => ({

6
src-tauri/vendored/plugins/importer-yaak/build/index.js generated
View File

@@ -69,6 +69,12 @@ function migrateImport(contents) {
}
}
}
for (const environment of parsed.resources.environments ?? []) {
if ("environmentId" in environment) {
environment.base = environment.environmentId == null;
delete environment.environmentId;
}
}
return { resources: parsed.resources };
}
function isJSObject(obj) {

2
src-tauri/yaak-crypto/src/encryption.rs
View File

@@ -38,7 +38,7 @@ pub(crate) fn decrypt_data(cipher_data: &[u8], key: &Key<XChaCha20Poly1305>) ->
let (nonce, ciphered_data) = rest.split_at_checked(nonce_bytes).ok_or(InvalidEncryptedData)?;
let cipher = XChaCha20Poly1305::new(&key);
cipher.decrypt(nonce.into(), ciphered_data).map_err(|_| DecryptionError)
cipher.decrypt(nonce.into(), ciphered_data).map_err(|_e| DecryptionError)
}
#[cfg(test)]

7
src-tauri/yaak-crypto/src/error.rs
View File

@@ -16,13 +16,16 @@ pub enum Error {
#[error("Incorrect workspace key")]
IncorrectWorkspaceKey,
#[error("Failed to decrypt workspace key: {0}")]
WorkspaceKeyDecryptionError(String),
#[error("Crypto IO error: {0}")]
IoError(#[from] io::Error),
#[error("Failed to encrypt")]
#[error("Failed to encrypt data")]
EncryptionError,
#[error("Failed to decrypt")]
#[error("Failed to decrypt data")]
DecryptionError,
#[error("Invalid encrypted data")]

10
src-tauri/yaak-crypto/src/manager.rs
View File

@@ -1,4 +1,6 @@
use crate::error::Error::{GenericError, IncorrectWorkspaceKey, MissingWorkspaceKey};
use crate::error::Error::{
GenericError, IncorrectWorkspaceKey, MissingWorkspaceKey, WorkspaceKeyDecryptionError,
};
use crate::error::{Error, Result};
use crate::master_key::MasterKey;
use crate::workspace_key::WorkspaceKey;
@@ -149,8 +151,10 @@ impl EncryptionManager {
let mkey = self.get_master_key()?;
let decoded_key = BASE64_STANDARD
.decode(key.encrypted_key)
.map_err(|e| GenericError(format!("Failed to decode workspace key {e:?}")))?;
let raw_key = mkey.decrypt(decoded_key.as_slice())?;
.map_err(|e| WorkspaceKeyDecryptionError(e.to_string()))?;
let raw_key = mkey
.decrypt(decoded_key.as_slice())
.map_err(|e| WorkspaceKeyDecryptionError(e.to_string()))?;
info!("Got existing workspace key for {workspace_id}");
let wkey = WorkspaceKey::from_raw_key(raw_key.as_slice());

2
src-tauri/yaak-git/bindings/gen_models.ts
View File

@@ -1,6 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type Environment = { model: "environment", id: string, workspaceId: string, environmentId: string | null, createdAt: string, updatedAt: string, name: string, variables: Array<EnvironmentVariable>, };
export type Environment = { model: "environment", id: string, workspaceId: string, createdAt: string, updatedAt: string, name: string, public: boolean, base: boolean, variables: Array<EnvironmentVariable>, };
export type EnvironmentVariable = { enabled?: boolean, name: string, value: string, id?: string, };

2
src-tauri/yaak-models/bindings/gen_models.ts
View File

@@ -14,7 +14,7 @@ export type EditorKeymap = "default" | "vim" | "vscode" | "emacs";
export type EncryptedKey = { encryptedKey: string, };
export type Environment = { model: "environment", id: string, workspaceId: string, environmentId: string | null, createdAt: string, updatedAt: string, name: string, variables: Array<EnvironmentVariable>, };
export type Environment = { model: "environment", id: string, workspaceId: string, createdAt: string, updatedAt: string, name: string, public: boolean, base: boolean, variables: Array<EnvironmentVariable>, };
export type EnvironmentVariable = { enabled?: boolean, name: string, value: string, id?: string, };

2
src-tauri/yaak-models/src/commands.rs
View File

@@ -109,7 +109,7 @@ pub(crate) fn workspace_models<R: Runtime>(
// Add the workspace children
if let Some(wid) = workspace_id {
l.append(&mut db.list_cookie_jars(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_environments(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_environments_ensure_base(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_folders(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_grpc_connections(wid)?.into_iter().map(Into::into).collect());
l.append(&mut db.list_grpc_requests(wid)?.into_iter().map(Into::into).collect());

6
src-tauri/yaak-models/src/error.rs
View File

@@ -21,6 +21,12 @@ pub enum Error {
#[error("Model error: {0}")]
GenericError(String),
#[error("No base environment for {0}")]
MissingBaseEnvironment(String),
#[error("Multiple base environments for {0}. Delete duplicates before continuing.")]
MultipleBaseEnvironments(String),
#[error("Row not found")]
RowNotFound,

11
src-tauri/yaak-models/src/models.rs
View File

@@ -486,11 +486,12 @@ pub struct Environment {
pub model: String,
pub id: String,
pub workspace_id: String,
pub environment_id: Option<String>,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
pub name: String,
pub public: bool,
pub base: bool,
pub variables: Vec<EnvironmentVariable>,
}
@@ -523,9 +524,10 @@ impl UpsertModelInfo for Environment {
Ok(vec![
(CreatedAt, upsert_date(source, self.created_at)),
(UpdatedAt, upsert_date(source, self.updated_at)),
(EnvironmentId, self.environment_id.into()),
(WorkspaceId, self.workspace_id.into()),
(Base, self.base.into()),
(Name, self.name.trim().into()),
(Public, self.public.into()),
(Variables, serde_json::to_string(&self.variables)?.into()),
])
}
@@ -533,7 +535,9 @@ impl UpsertModelInfo for Environment {
fn update_columns() -> Vec<impl IntoIden> {
vec![
EnvironmentIden::UpdatedAt,
EnvironmentIden::Base,
EnvironmentIden::Name,
EnvironmentIden::Public,
EnvironmentIden::Variables,
]
}
@@ -547,10 +551,11 @@ impl UpsertModelInfo for Environment {
id: row.get("id")?,
model: row.get("model")?,
workspace_id: row.get("workspace_id")?,
environment_id: row.get("environment_id")?,
created_at: row.get("created_at")?,
updated_at: row.get("updated_at")?,
base: row.get("base")?,
name: row.get("name")?,
public: row.get("public")?,
variables: serde_json::from_str(variables.as_str()).unwrap_or_default(),
})
}

65
src-tauri/yaak-models/src/queries/environments.rs
View File

@@ -1,8 +1,9 @@
use crate::db_context::DbContext;
use crate::error::Error::GenericError;
use crate::error::Error::{MissingBaseEnvironment, MultipleBaseEnvironments};
use crate::error::Result;
use crate::models::{Environment, EnvironmentIden};
use crate::models::{Environment, EnvironmentIden, EnvironmentVariable};
use crate::util::UpdateSource;
use log::info;
impl<'a> DbContext<'a> {
pub fn get_environment(&self, id: &str) -> Result<Environment> {
@@ -10,35 +11,41 @@ impl<'a> DbContext<'a> {
}
pub fn get_base_environment(&self, workspace_id: &str) -> Result<Environment> {
// Will create base environment if it doesn't exist
let environments = self.list_environments(workspace_id)?;
let environments = self.list_environments_ensure_base(workspace_id)?;
let base_environments =
environments.into_iter().filter(|e| e.base).collect::<Vec<Environment>>();
let base_environment = environments
.into_iter()
.find(|e| e.environment_id == None && e.workspace_id == workspace_id)
.ok_or(GenericError(format!("No base environment found for {workspace_id}")))?;
if base_environments.len() > 1 {
return Err(MultipleBaseEnvironments(workspace_id.to_string()));
}
let base_environment = base_environments.into_iter().find(|e| e.base).ok_or(
// Should never happen because one should be created above if it does not exist
MissingBaseEnvironment(workspace_id.to_string()),
)?;
Ok(base_environment)
}
pub fn list_environments(&self, workspace_id: &str) -> Result<Vec<Environment>> {
/// Lists environments and will create a base environment if one doesn't exist
pub fn list_environments_ensure_base(&self, workspace_id: &str) -> Result<Vec<Environment>> {
let mut environments =
self.find_many::<Environment>(EnvironmentIden::WorkspaceId, workspace_id, None)?;
let base_environment = environments
.iter()
.find(|e| e.environment_id == None && e.workspace_id == workspace_id);
let base_environment = environments.iter().find(|e| e.base);
if let None = base_environment {
environments.push(self.upsert_environment(
let e = self.upsert_environment(
&Environment {
workspace_id: workspace_id.to_string(),
environment_id: None,
base: true,
name: "Global Variables".to_string(),
..Default::default()
},
&UpdateSource::Background,
)?);
)?;
info!("Created base environment {} for {workspace_id}", e.id);
environments.push(e);
}
Ok(environments)
@@ -49,12 +56,12 @@ impl<'a> DbContext<'a> {
environment: &Environment,
source: &UpdateSource,
) -> Result<Environment> {
for environment in
self.find_many::<Environment>(EnvironmentIden::EnvironmentId, &environment.id, None)?
{
self.delete_environment(&environment, source)?;
}
self.delete(environment, source)
let deleted_environment = self.delete(environment, source)?;
// Recreate the base environment if we happened to delete it
self.list_environments_ensure_base(&environment.workspace_id)?;
Ok(deleted_environment)
}
pub fn delete_environment_by_id(&self, id: &str, source: &UpdateSource) -> Result<Environment> {
@@ -69,7 +76,7 @@ impl<'a> DbContext<'a> {
) -> Result<Environment> {
let mut environment = environment.clone();
environment.id = "".to_string();
self.upsert(&environment, source)
self.upsert_environment(&environment, source)
}
pub fn upsert_environment(
@@ -77,6 +84,18 @@ impl<'a> DbContext<'a> {
environment: &Environment,
source: &UpdateSource,
) -> Result<Environment> {
self.upsert(environment, source)
let cleaned_variables = environment
.variables
.iter()
.filter(|v| !v.name.is_empty() || !v.value.is_empty())
.cloned()
.collect::<Vec<EnvironmentVariable>>();
self.upsert(
&Environment {
variables: cleaned_variables,
..environment.clone()
},
source,
)
}
}

8
src-tauri/yaak-models/src/queries/workspace_metas.rs
View File

@@ -2,6 +2,7 @@ use crate::db_context::DbContext;
use crate::error::Result;
use crate::models::{WorkspaceMeta, WorkspaceMetaIden};
use crate::util::UpdateSource;
use log::info;
impl<'a> DbContext<'a> {
pub fn get_workspace_meta(&self, workspace_id: &str) -> Option<WorkspaceMeta> {
@@ -23,10 +24,7 @@ impl<'a> DbContext<'a> {
Ok(workspace_metas)
}
pub fn get_or_create_workspace_meta(
&self,
workspace_id: &str,
) -> Result<WorkspaceMeta> {
pub fn get_or_create_workspace_meta(&self, workspace_id: &str) -> Result<WorkspaceMeta> {
let workspace_meta = self.get_workspace_meta(workspace_id);
if let Some(workspace_meta) = workspace_meta {
return Ok(workspace_meta);
@@ -37,6 +35,8 @@ impl<'a> DbContext<'a> {
..Default::default()
};
info!("Creating WorkspaceMeta for {workspace_id}");
self.upsert_workspace_meta(&workspace_meta, &UpdateSource::Background)
}

28
src-tauri/yaak-models/src/queries/workspaces.rs
View File

@@ -1,8 +1,8 @@
use crate::db_context::DbContext;
use crate::error::Result;
use crate::models::{
Folder, FolderIden, GrpcRequest, GrpcRequestIden, HttpRequest, HttpRequestIden,
WebsocketRequest, WebsocketRequestIden, Workspace, WorkspaceIden,
EnvironmentIden, FolderIden, GrpcRequestIden, HttpRequestIden, WebsocketRequestIden, Workspace,
WorkspaceIden,
};
use crate::util::UpdateSource;
@@ -34,24 +34,26 @@ impl<'a> DbContext<'a> {
workspace: &Workspace,
source: &UpdateSource,
) -> Result<Workspace> {
for m in self.find_many::<HttpRequest>(HttpRequestIden::WorkspaceId, &workspace.id, None)? {
for m in self.find_many(HttpRequestIden::WorkspaceId, &workspace.id, None)? {
self.delete_http_request(&m, source)?;
}
for m in self.find_many::<GrpcRequest>(GrpcRequestIden::WorkspaceId, &workspace.id, None)? {
for m in self.find_many(GrpcRequestIden::WorkspaceId, &workspace.id, None)? {
self.delete_grpc_request(&m, source)?;
}
for m in
self.find_many::<WebsocketRequest>(WebsocketRequestIden::FolderId, &workspace.id, None)?
{
for m in self.find_many(WebsocketRequestIden::FolderId, &workspace.id, None)? {
self.delete_websocket_request(&m, source)?;
}
for folder in self.find_many::<Folder>(FolderIden::WorkspaceId, &workspace.id, None)? {
self.delete_folder(&folder, source)?;
for m in self.find_many(FolderIden::WorkspaceId, &workspace.id, None)? {
self.delete_folder(&m, source)?;
}
for m in self.find_many(EnvironmentIden::WorkspaceId, &workspace.id, None)? {
self.delete_environment(&m, source)?;
}
self.delete(workspace, source)
}

24
src-tauri/yaak-models/src/util.rs
View File

@@ -1,5 +1,8 @@
use crate::error::Result;
use crate::models::{AnyModel, Environment, Folder, GrpcRequest, HttpRequest, UpsertModelInfo, WebsocketRequest, Workspace, WorkspaceIden};
use crate::models::{
AnyModel, Environment, Folder, GrpcRequest, HttpRequest, UpsertModelInfo, WebsocketRequest,
Workspace, WorkspaceIden,
};
use crate::query_manager::QueryManagerExt;
use chrono::{NaiveDateTime, Utc};
use log::warn;
@@ -117,14 +120,14 @@ pub struct BatchUpsertResult {
pub websocket_requests: Vec<WebsocketRequest>,
}
pub async fn get_workspace_export_resources<R: Runtime>(
pub fn get_workspace_export_resources<R: Runtime>(
app_handle: &AppHandle<R>,
workspace_ids: Vec<&str>,
include_environments: bool,
include_private_environments: bool,
) -> Result<WorkspaceExport> {
let mut data = WorkspaceExport {
yaak_version: app_handle.package_info().version.clone().to_string(),
yaak_schema: 3,
yaak_schema: 4,
timestamp: Utc::now().naive_utc(),
resources: BatchUpsertResult {
workspaces: Vec::new(),
@@ -139,18 +142,19 @@ pub async fn get_workspace_export_resources<R: Runtime>(
let db = app_handle.db();
for workspace_id in workspace_ids {
data.resources.workspaces.push(db.find_one(WorkspaceIden::Id, workspace_id)?);
data.resources.environments.append(&mut db.list_environments(workspace_id)?);
data.resources.environments.append(
&mut db
.list_environments_ensure_base(workspace_id)?
.into_iter()
.filter(|e| include_private_environments || e.public)
.collect(),
);
data.resources.folders.append(&mut db.list_folders(workspace_id)?);
data.resources.http_requests.append(&mut db.list_http_requests(workspace_id)?);
data.resources.grpc_requests.append(&mut db.list_grpc_requests(workspace_id)?);
data.resources.websocket_requests.append(&mut db.list_websocket_requests(workspace_id)?);
}
// Nuke environments if we don't want them
if !include_environments {
data.resources.environments.clear();
}
Ok(data)
}

2
src-tauri/yaak-plugins/bindings/gen_models.ts
View File

@@ -1,6 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type Environment = { model: "environment", id: string, workspaceId: string, environmentId: string | null, createdAt: string, updatedAt: string, name: string, variables: Array<EnvironmentVariable>, };
export type Environment = { model: "environment", id: string, workspaceId: string, createdAt: string, updatedAt: string, name: string, public: boolean, base: boolean, variables: Array<EnvironmentVariable>, };
export type EnvironmentVariable = { enabled?: boolean, name: string, value: string, id?: string, };

2
src-tauri/yaak-sync/bindings/gen_models.ts
View File

@@ -1,6 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type Environment = { model: "environment", id: string, workspaceId: string, environmentId: string | null, createdAt: string, updatedAt: string, name: string, variables: Array<EnvironmentVariable>, };
export type Environment = { model: "environment", id: string, workspaceId: string, createdAt: string, updatedAt: string, name: string, public: boolean, base: boolean, variables: Array<EnvironmentVariable>, };
export type EnvironmentVariable = { enabled?: boolean, name: string, value: string, id?: string, };

2
src-tauri/yaak-sync/bindings/gen_sync.ts
View File

@@ -4,4 +4,4 @@ import type { SyncState } from "./gen_models.js";
export type FsCandidate = { "type": "FsCandidate", model: SyncModel, relPath: string, checksum: string, };
export type SyncOp = { "type": "fsCreate", model: SyncModel, } | { "type": "fsUpdate", model: SyncModel, state: SyncState, } | { "type": "fsDelete", state: SyncState, fs: FsCandidate | null, } | { "type": "dbCreate", fs: FsCandidate, } | { "type": "dbUpdate", state: SyncState, fs: FsCandidate, } | { "type": "dbDelete", model: SyncModel, state: SyncState, };
export type SyncOp = { "type": "fsCreate", model: SyncModel, } | { "type": "fsUpdate", model: SyncModel, state: SyncState, } | { "type": "fsDelete", state: SyncState, fs: FsCandidate | null, } | { "type": "dbCreate", fs: FsCandidate, } | { "type": "dbUpdate", state: SyncState, fs: FsCandidate, } | { "type": "dbDelete", model: SyncModel, state: SyncState, } | { "type": "ignorePrivate", model: SyncModel, };

15
src-tauri/yaak-sync/src/commands.rs
View File

@@ -1,7 +1,7 @@
use crate::error::Result;
use crate::sync::{
apply_sync_ops, apply_sync_state_ops, compute_sync_ops, get_db_candidates, get_fs_candidates,
FsCandidate, SyncOp,
apply_sync_ops, apply_sync_state_ops, compute_sync_ops, get_db_candidates, get_fs_candidates, FsCandidate,
SyncOp,
};
use crate::watch::{watch_directory, WatchEvent};
use chrono::Utc;
@@ -19,9 +19,8 @@ pub async fn calculate<R: Runtime>(
workspace_id: &str,
sync_dir: &Path,
) -> Result<Vec<SyncOp>> {
let db_candidates = get_db_candidates(&app_handle, workspace_id, sync_dir).await?;
let fs_candidates = get_fs_candidates(sync_dir)
.await?
let db_candidates = get_db_candidates(&app_handle, workspace_id, sync_dir)?;
let fs_candidates = get_fs_candidates(sync_dir)?
.into_iter()
// Only keep items in the same workspace
.filter(|fs| fs.model.workspace_id() == workspace_id)
@@ -34,7 +33,7 @@ pub async fn calculate<R: Runtime>(
#[command]
pub async fn calculate_fs(dir: &Path) -> Result<Vec<SyncOp>> {
let db_candidates = Vec::new();
let fs_candidates = get_fs_candidates(dir).await?;
let fs_candidates = get_fs_candidates(dir)?;
Ok(compute_sync_ops(db_candidates, fs_candidates))
}
@@ -45,8 +44,8 @@ pub async fn apply<R: Runtime>(
sync_dir: &Path,
workspace_id: &str,
) -> Result<()> {
let sync_state_ops = apply_sync_ops(&app_handle, &workspace_id, sync_dir, sync_ops).await?;
apply_sync_state_ops(&app_handle, workspace_id, sync_dir, sync_state_ops).await
let sync_state_ops = apply_sync_ops(&app_handle, &workspace_id, sync_dir, sync_ops)?;
apply_sync_state_ops(&app_handle, workspace_id, sync_dir, sync_state_ops)
}
#[derive(Debug, Clone, Serialize, Deserialize, TS)]

2
src-tauri/yaak-sync/src/models.rs
View File

@@ -130,4 +130,4 @@ impl TryFrom<AnyModel> for SyncModel {
};
Ok(m)
}
}
}

207
src-tauri/yaak-sync/src/sync.rs
View File

@@ -1,15 +1,15 @@
use crate::error::Result;
use crate::models::SyncModel;
use chrono::Utc;
use log::{debug, info, warn};
use log::{info, warn};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::fs;
use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};
use tauri::{AppHandle, Runtime};
use tokio::fs;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use ts_rs::TS;
use yaak_models::models::{SyncState, WorkspaceMeta};
use yaak_models::query_manager::QueryManagerExt;
@@ -41,17 +41,21 @@ pub(crate) enum SyncOp {
model: SyncModel,
state: SyncState,
},
IgnorePrivate {
model: SyncModel,
},
}
impl SyncOp {
fn workspace_id(&self) -> String {
match self {
SyncOp::FsCreate { model } => model.workspace_id(),
SyncOp::FsUpdate { state, .. } => state.workspace_id.clone(),
SyncOp::FsDelete { state, .. } => state.workspace_id.clone(),
SyncOp::DbCreate { fs } => fs.model.workspace_id(),
SyncOp::DbUpdate { state, .. } => state.workspace_id.clone(),
SyncOp::DbDelete { model, .. } => model.workspace_id(),
SyncOp::DbUpdate { state, .. } => state.workspace_id.clone(),
SyncOp::FsCreate { model } => model.workspace_id(),
SyncOp::FsDelete { state, .. } => state.workspace_id.clone(),
SyncOp::FsUpdate { state, .. } => state.workspace_id.clone(),
SyncOp::IgnorePrivate { model } => model.workspace_id(),
}
}
}
@@ -66,6 +70,7 @@ impl Display for SyncOp {
SyncOp::DbCreate { fs } => format!("db_create({})", fs.model.id()),
SyncOp::DbUpdate { fs, .. } => format!("db_update({})", fs.model.id()),
SyncOp::DbDelete { model, .. } => format!("db_delete({})", model.id()),
SyncOp::IgnorePrivate { model } => format!("ignore_private({})", model.id()),
}
.as_str(),
)
@@ -76,8 +81,8 @@ impl Display for SyncOp {
#[serde(rename_all = "snake_case")]
pub(crate) enum DbCandidate {
Added(SyncModel),
Modified(SyncModel, SyncState),
Deleted(SyncState),
Modified(SyncModel, SyncState),
Unmodified(SyncModel, SyncState),
}
@@ -85,8 +90,8 @@ impl DbCandidate {
fn model_id(&self) -> String {
match &self {
DbCandidate::Added(m) => m.id(),
DbCandidate::Modified(m, _) => m.id(),
DbCandidate::Deleted(s) => s.model_id.clone(),
DbCandidate::Modified(m, _) => m.id(),
DbCandidate::Unmodified(m, _) => m.id(),
}
}
@@ -101,16 +106,13 @@ pub(crate) struct FsCandidate {
pub(crate) checksum: String,
}
pub(crate) async fn get_db_candidates<R: Runtime>(
pub(crate) fn get_db_candidates<R: Runtime>(
app_handle: &AppHandle<R>,
workspace_id: &str,
sync_dir: &Path,
) -> Result<Vec<DbCandidate>> {
let models: HashMap<_, _> = workspace_models(app_handle, workspace_id)
.await?
.into_iter()
.map(|m| (m.id(), m))
.collect();
let models: HashMap<_, _> =
workspace_models(app_handle, workspace_id)?.into_iter().map(|m| (m.id(), m)).collect();
let sync_states: HashMap<_, _> = app_handle
.db()
.list_sync_states_for_workspace(workspace_id, sync_dir)?
@@ -121,20 +123,42 @@ pub(crate) async fn get_db_candidates<R: Runtime>(
// 1. Add candidates for models (created/modified/unmodified)
let mut candidates: Vec<DbCandidate> = models
.values()
.map(|model| {
let existing_sync_state = match sync_states.get(&model.id()) {
Some(s) => s,
None => {
// No sync state yet, so model was just added
return DbCandidate::Added(model.to_owned());
}
};
.filter_map(|model| {
match sync_states.get(&model.id()) {
Some(existing_sync_state) => {
// If a sync state exists but the model is now private, treat it as a deletion
match model {
SyncModel::Environment(e) if !e.public => {
return Some(DbCandidate::Deleted(existing_sync_state.to_owned()));
}
_ => {}
};
let updated_since_flush = model.updated_at() > existing_sync_state.flushed_at;
if updated_since_flush {
DbCandidate::Modified(model.to_owned(), existing_sync_state.to_owned())
} else {
DbCandidate::Unmodified(model.to_owned(), existing_sync_state.to_owned())
let updated_since_flush = model.updated_at() > existing_sync_state.flushed_at;
if updated_since_flush {
Some(DbCandidate::Modified(
model.to_owned(),
existing_sync_state.to_owned(),
))
} else {
Some(DbCandidate::Unmodified(
model.to_owned(),
existing_sync_state.to_owned(),
))
}
}
None => {
return match model {
SyncModel::Environment(e) if !e.public => {
// No sync state yet, so ignore the model
None
}
_ => {
// No sync state yet, so the model was just added
Some(DbCandidate::Added(model.to_owned()))
}
};
}
}
})
.collect();
@@ -151,30 +175,28 @@ pub(crate) async fn get_db_candidates<R: Runtime>(
Ok(candidates)
}
pub(crate) async fn get_fs_candidates(dir: &Path) -> Result<Vec<FsCandidate>> {
pub(crate) fn get_fs_candidates(dir: &Path) -> Result<Vec<FsCandidate>> {
// Ensure the root directory exists
fs::create_dir_all(dir).await?;
fs::create_dir_all(dir)?;
let mut candidates = Vec::new();
let mut entries = fs::read_dir(dir).await?;
while let Some(dir_entry) = entries.next_entry().await? {
if !dir_entry.file_type().await?.is_file() {
let entries = fs::read_dir(dir)?;
for dir_entry in entries {
let dir_entry = match dir_entry {
Ok(v) => v,
Err(_) => continue,
};
if !dir_entry.file_type()?.is_file() {
continue;
};
let path = dir_entry.path();
let (model, checksum) = match SyncModel::from_file(&path) {
// TODO: Remove this once we have logic to handle environments. This it to clean
// any existing ones from the sync dir that resulted from the 2025.1 betas.
Ok(Some((SyncModel::Environment(e), _))) => {
fs::remove_file(path).await?;
info!("Cleaned up synced environment {}", e.id);
continue;
}
Ok(Some(m)) => m,
Ok(None) => continue,
Err(e) => {
warn!("Failed to read sync file {e}");
warn!("Failed to parse sync file {e}");
return Err(e);
}
};
@@ -212,32 +234,32 @@ pub(crate) fn compute_sync_ops(
let op = match (db_map.get(k), fs_map.get(k)) {
(None, None) => return None, // Can never happen
(None, Some(fs)) => SyncOp::DbCreate { fs: fs.to_owned() },
(Some(DbCandidate::Unmodified(model, sync_state)), None) => {
// TODO: Remove this once we have logic to handle environments. This it to
// ignore the cleaning we did above of any environments that were written
// to disk in the 2025.1 betas.
if let SyncModel::Environment(_) = model {
return None;
}
SyncOp::DbDelete {
model: model.to_owned(),
state: sync_state.to_owned(),
}
}
// DB unchanged <-> FS missing
(Some(DbCandidate::Unmodified(model, sync_state)), None) => SyncOp::DbDelete {
model: model.to_owned(),
state: sync_state.to_owned(),
},
// DB modified <-> FS missing
(Some(DbCandidate::Modified(model, sync_state)), None) => SyncOp::FsUpdate {
model: model.to_owned(),
state: sync_state.to_owned(),
},
// DB added <-> FS missing
(Some(DbCandidate::Added(model)), None) => SyncOp::FsCreate {
model: model.to_owned(),
},
(Some(DbCandidate::Deleted(sync_state)), None) => {
// Already deleted on FS, but sending it so the SyncState gets dealt with
SyncOp::FsDelete {
state: sync_state.to_owned(),
fs: None,
}
}
// DB deleted <-> FS missing
// Already deleted on FS, but sending it so the SyncState gets dealt with
(Some(DbCandidate::Deleted(sync_state)), None) => SyncOp::FsDelete {
state: sync_state.to_owned(),
fs: None,
},
// DB unchanged <-> FS exists
(Some(DbCandidate::Unmodified(_, sync_state)), Some(fs_candidate)) => {
if sync_state.checksum == fs_candidate.checksum {
return None;
@@ -248,6 +270,8 @@ pub(crate) fn compute_sync_ops(
}
}
}
// DB modified <-> FS exists
(Some(DbCandidate::Modified(model, sync_state)), Some(fs_candidate)) => {
if sync_state.checksum == fs_candidate.checksum {
SyncOp::FsUpdate {
@@ -255,25 +279,29 @@ pub(crate) fn compute_sync_ops(
state: sync_state.to_owned(),
}
} else if model.updated_at() < fs_candidate.model.updated_at() {
// CONFLICT! Write to DB if fs model is newer
// CONFLICT! Write to DB if the fs model is newer
SyncOp::DbUpdate {
state: sync_state.to_owned(),
fs: fs_candidate.to_owned(),
}
} else {
// CONFLICT! Write to FS if db model is newer
// CONFLICT! Write to FS if the db model is newer
SyncOp::FsUpdate {
model: model.to_owned(),
state: sync_state.to_owned(),
}
}
}
// DB added <-> FS anything
(Some(DbCandidate::Added(model)), Some(_)) => {
// This would be super rare (impossible?), so let's follow the user's intention
SyncOp::FsCreate {
model: model.to_owned(),
}
}
// DB deleted <-> FS exists
(Some(DbCandidate::Deleted(sync_state)), Some(fs_candidate)) => SyncOp::FsDelete {
state: sync_state.to_owned(),
fs: Some(fs_candidate.to_owned()),
@@ -284,12 +312,19 @@ pub(crate) fn compute_sync_ops(
.collect()
}
async fn workspace_models<R: Runtime>(
fn workspace_models<R: Runtime>(
app_handle: &AppHandle<R>,
workspace_id: &str,
) -> Result<Vec<SyncModel>> {
let resources =
get_workspace_export_resources(app_handle, vec![workspace_id], true).await?.resources;
// We want to include private environments here so that we can take them into account during
// the sync process. Otherwise, they would be treated as deleted.
let include_private_environments = true;
let resources = get_workspace_export_resources(
app_handle,
vec![workspace_id],
include_private_environments,
)?
.resources;
let workspace = resources.workspaces.iter().find(|w| w.id == workspace_id);
let workspace = match workspace {
@@ -318,7 +353,7 @@ async fn workspace_models<R: Runtime>(
Ok(sync_models)
}
pub(crate) async fn apply_sync_ops<R: Runtime>(
pub(crate) fn apply_sync_ops<R: Runtime>(
app_handle: &AppHandle<R>,
workspace_id: &str,
sync_dir: &Path,
@@ -328,13 +363,14 @@ pub(crate) async fn apply_sync_ops<R: Runtime>(
return Ok(Vec::new());
}
debug!(
info!(
"Applying sync ops {}",
sync_ops.iter().map(|op| op.to_string()).collect::<Vec<String>>().join(", ")
);
let mut sync_state_ops = Vec::new();
let mut workspaces_to_upsert = Vec::new();
let environments_to_upsert = Vec::new();
let mut environments_to_upsert = Vec::new();
let mut folders_to_upsert = Vec::new();
let mut http_requests_to_upsert = Vec::new();
let mut grpc_requests_to_upsert = Vec::new();
@@ -351,8 +387,8 @@ pub(crate) async fn apply_sync_ops<R: Runtime>(
let rel_path = derive_model_filename(&model);
let abs_path = sync_dir.join(rel_path.clone());
let (content, checksum) = model.to_file_contents(&rel_path)?;
let mut f = File::create(&abs_path).await?;
f.write_all(&content).await?;
let mut f = File::create(&abs_path)?;
f.write_all(&content)?;
SyncStateOp::Create {
model_id: model.id(),
checksum,
@@ -364,8 +400,8 @@ pub(crate) async fn apply_sync_ops<R: Runtime>(
let rel_path = Path::new(&state.rel_path);
let abs_path = Path::new(&state.sync_dir).join(&rel_path);
let (content, checksum) = model.to_file_contents(&rel_path)?;
let mut f = File::create(&abs_path).await?;
f.write_all(&content).await?;
let mut f = File::create(&abs_path)?;
f.write_all(&content)?;
SyncStateOp::Update {
state: state.to_owned(),
checksum,
@@ -383,7 +419,7 @@ pub(crate) async fn apply_sync_ops<R: Runtime>(
// Always delete the existing path
let rel_path = Path::new(&state.rel_path);
let abs_path = Path::new(&state.sync_dir).join(&rel_path);
fs::remove_file(&abs_path).await?;
fs::remove_file(&abs_path)?;
SyncStateOp::Delete {
state: state.to_owned(),
}
@@ -395,14 +431,12 @@ pub(crate) async fn apply_sync_ops<R: Runtime>(
// Push updates to arrays so we can do them all in a single
// batch upsert to make foreign keys happy
match fs.model {
SyncModel::Workspace(m) => workspaces_to_upsert.push(m),
SyncModel::Environment(m) => environments_to_upsert.push(m),
SyncModel::Folder(m) => folders_to_upsert.push(m),
SyncModel::HttpRequest(m) => http_requests_to_upsert.push(m),
SyncModel::GrpcRequest(m) => grpc_requests_to_upsert.push(m),
SyncModel::HttpRequest(m) => http_requests_to_upsert.push(m),
SyncModel::WebsocketRequest(m) => websocket_requests_to_upsert.push(m),
// TODO: Handle environments in sync
SyncModel::Environment(_) => {}
SyncModel::Workspace(m) => workspaces_to_upsert.push(m),
};
SyncStateOp::Create {
model_id,
@@ -414,14 +448,12 @@ pub(crate) async fn apply_sync_ops<R: Runtime>(
// Push updates to arrays so we can do them all in a single
// batch upsert to make foreign keys happy
match fs.model {
SyncModel::Workspace(m) => workspaces_to_upsert.push(m),
SyncModel::Environment(m) => environments_to_upsert.push(m),
SyncModel::Folder(m) => folders_to_upsert.push(m),
SyncModel::HttpRequest(m) => http_requests_to_upsert.push(m),
SyncModel::GrpcRequest(m) => grpc_requests_to_upsert.push(m),
SyncModel::HttpRequest(m) => http_requests_to_upsert.push(m),
SyncModel::WebsocketRequest(m) => websocket_requests_to_upsert.push(m),
// TODO: Handle environments in sync
SyncModel::Environment(_) => {}
SyncModel::Workspace(m) => workspaces_to_upsert.push(m),
}
SyncStateOp::Update {
state: state.to_owned(),
@@ -435,6 +467,7 @@ pub(crate) async fn apply_sync_ops<R: Runtime>(
state: state.to_owned(),
}
}
SyncOp::IgnorePrivate { .. } => SyncStateOp::NoOp,
});
}
@@ -497,9 +530,10 @@ pub(crate) enum SyncStateOp {
Delete {
state: SyncState,
},
NoOp,
}
pub(crate) async fn apply_sync_state_ops<R: Runtime>(
pub(crate) fn apply_sync_state_ops<R: Runtime>(
app_handle: &AppHandle<R>,
workspace_id: &str,
sync_dir: &Path,
@@ -540,6 +574,9 @@ pub(crate) async fn apply_sync_state_ops<R: Runtime>(
SyncStateOp::Delete { state } => {
app_handle.db().delete_sync_state(&state)?;
}
SyncStateOp::NoOp => {
// Nothing
}
}
}
Ok(())