Optimize directory sync performance

src-tauri/Cargo.lock

@@ -6585,9 +6585,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.39.3"
+version = "1.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5"
+checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"
 dependencies = [
  "backtrace",
  "bytes",
@@ -8110,6 +8110,7 @@ dependencies = [
  "tauri",
  "tauri-plugin",
  "thiserror 2.0.7",
+ "tokio",
  "ts-rs",
  "yaak-models",
 ]

src-tauri/yaak-sync/Cargo.toml

@@ -17,6 +17,7 @@ log = "0.4.22"
 serde_json = "1.0.132"
 hex = "0.4.3"
 sha1 = "0.10.6"
+tokio = {version = "1.42.0", features = ["fs"]}
 
 [build-dependencies]
 tauri-plugin = { version = "2.0.3", features = ["build"] }

src-tauri/yaak-sync/index.ts

@@ -11,10 +11,11 @@ export const calculateSync = async (workspace: Workspace) => {
   });
 };
 
-export const applySync = async (workspace: Workspace, ops: SyncOp[]) => {
+export const applySync = async (workspace: Workspace, syncOps: SyncOp[]) => {
-  console.log('Applying sync', ops);
+  console.log('Applying sync', syncOps);
   return invoke<void>('plugin:yaak-sync|apply', {
     workspaceId: workspace.id,
     dir: workspace.settingSyncDir,
+    syncOps: syncOps
   });
 };

src-tauri/yaak-sync/src/commands.rs

@@ -3,11 +3,18 @@ use crate::sync::{apply_sync, calculate_sync, SyncOp};
 use tauri::{command, Runtime, WebviewWindow};
 
 #[command]
-pub async fn apply<R: Runtime>(window: WebviewWindow<R>, workspace_id: &str) -> Result<()> {
+pub async fn calculate<R: Runtime>(
-    apply_sync(&window, workspace_id).await
+    window: WebviewWindow<R>,
+    workspace_id: &str,
+) -> Result<Vec<SyncOp>> {
+    calculate_sync(&window, workspace_id).await
 }
 
 #[command]
-pub async fn calculate<R: Runtime>(window: WebviewWindow<R>, workspace_id: &str) -> Result<Vec<SyncOp>> {
+pub async fn apply<R: Runtime>(
-    calculate_sync(&window, workspace_id).await
+    window: WebviewWindow<R>,
+    sync_ops: Vec<SyncOp>,
+    workspace_id: &str,
+) -> Result<()> {
+    apply_sync(&window, workspace_id, sync_ops).await
 }

src-tauri/yaak-sync/src/models.rs

@@ -3,8 +3,8 @@ use crate::error::Result;
 use chrono::NaiveDateTime;
 use serde::{Deserialize, Serialize};
 use sha1::{Digest, Sha1};
-use std::fs;
 use std::path::Path;
+use tokio::fs;
 use ts_rs::TS;
 use yaak_models::models::{AnyModel, Environment, Folder, GrpcRequest, HttpRequest, Workspace};
 
@@ -20,8 +20,8 @@ pub enum SyncModel {
 }
 
 impl SyncModel {
-    pub fn from_file(file_path: &Path) -> Result<Option<(SyncModel, Vec<u8>, String)>> {
+    pub async fn from_file(file_path: &Path) -> Result<Option<(SyncModel, Vec<u8>, String)>> {
-        let content = match fs::read(file_path) {
+        let content = match fs::read(file_path).await {
             Ok(c) => c,
             Err(_) => return Ok(None),
         };

src-tauri/yaak-sync/src/sync.rs

@@ -6,10 +6,11 @@ use log::{debug, warn};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt::{Display, Formatter};
-use std::fs;
-use std::fs::create_dir_all;
 use std::path::{Path, PathBuf};
 use tauri::{Manager, Runtime, WebviewWindow};
+use tokio::fs;
+use tokio::fs::File;
+use tokio::io::AsyncWriteExt;
 use ts_rs::TS;
 use yaak_models::models::{SyncState, Workspace};
 use yaak_models::queries::{
@@ -97,7 +98,7 @@ pub(crate) async fn calculate_sync<R: Runtime>(
 ) -> Result<Vec<SyncOp>> {
     let workspace = get_workspace(window, workspace_id).await?;
     let db_candidates = get_db_candidates(window, &workspace).await?;
-    let fs_candidates = get_fs_candidates(&workspace)?;
+    let fs_candidates = get_fs_candidates(&workspace).await?;
     let sync_ops = compute_sync_ops(db_candidates, fs_candidates);
 
     Ok(sync_ops)
@@ -106,10 +107,9 @@ pub(crate) async fn calculate_sync<R: Runtime>(
 pub(crate) async fn apply_sync<R: Runtime>(
     window: &WebviewWindow<R>,
     workspace_id: &str,
+    sync_ops: Vec<SyncOp>,
 ) -> Result<()> {
     let workspace = get_workspace(window, workspace_id).await?;
-    let sync_ops = calculate_sync(window, workspace_id).await?;
-
     let sync_state_ops = apply_sync_ops(window, &workspace, sync_ops).await?;
     let result = apply_sync_state_ops(window, &workspace, sync_state_ops).await;
 
@@ -120,17 +120,21 @@ async fn get_db_candidates<R: Runtime>(
     mgr: &impl Manager<R>,
     workspace: &Workspace,
 ) -> Result<Vec<DbCandidate>> {
-    let workspace_id = workspace.id.as_str();
-    let models = workspace_models(mgr, workspace).await;
     let sync_dir = get_workspace_sync_dir(workspace)?;
-    let sync_states = list_sync_states_for_workspace(mgr, workspace_id, sync_dir).await?;
+    let models: HashMap<_, _> =
+        workspace_models(mgr, workspace).await.into_iter().map(|m| (m.id(), m)).collect();
+    let sync_states: HashMap<_, _> =
+        list_sync_states_for_workspace(mgr, workspace.id.as_str(), sync_dir)
+            .await?
+            .into_iter()
+            .map(|s| (s.model_id.clone(), s))
+            .collect();
 
     // 1. Add candidates for models (created/modified/unmodified)
     let mut candidates: Vec<DbCandidate> = models
-        .iter()
+        .values()
         .map(|model| {
-            let existing_sync_state = sync_states.iter().find(|ss| ss.model_id == model.id());
+            let existing_sync_state = match sync_states.get(&model.id()) {
-            let existing_sync_state = match existing_sync_state {
                 Some(s) => s,
                 None => {
                     // No sync state yet, so model was just added
@@ -148,8 +152,8 @@ async fn get_db_candidates<R: Runtime>(
         .collect();
 
     // 2. Add SyncState-only candidates (deleted)
-    candidates.extend(sync_states.iter().filter_map(|sync_state| {
+    candidates.extend(sync_states.values().filter_map(|sync_state| {
-        let already_added = models.iter().find(|m| m.id() == sync_state.model_id).is_some();
+        let already_added = models.contains_key(&sync_state.model_id);
         if already_added {
             return None;
         }
@@ -159,47 +163,46 @@ async fn get_db_candidates<R: Runtime>(
     Ok(candidates)
 }
 
-fn get_fs_candidates(workspace: &Workspace) -> Result<Vec<FsCandidate>> {
+async fn get_fs_candidates(workspace: &Workspace) -> Result<Vec<FsCandidate>> {
     let dir = match workspace.setting_sync_dir.clone() {
         None => return Ok(Vec::new()),
         Some(d) => d,
     };
 
     // Ensure the root directory exists
-    create_dir_all(dir.clone())?;
+    fs::create_dir_all(dir.clone()).await?;
 
-    let candidates = fs::read_dir(dir)?
+    let mut candidates = Vec::new();
-        .filter_map(|dir_entry| {
+    let mut entries = fs::read_dir(dir).await?;
-            let dir_entry = dir_entry.ok()?;
+    while let Some(dir_entry) = entries.next_entry().await? {
-            if !dir_entry.file_type().ok()?.is_file() {
+        if !dir_entry.file_type().await?.is_file() {
-                return None;
+            continue;
-            };
+        };
 
-            let path = dir_entry.path();
+        let path = dir_entry.path();
-            let (model, _, checksum) = match SyncModel::from_file(&path) {
+        let (model, _, checksum) = match SyncModel::from_file(&path).await {
-                Ok(Some(m)) => m,
+            Ok(Some(m)) => m,
-                Ok(None) => return None,
+            Ok(None) => continue,
-                Err(InvalidSyncFile(_)) => return None,
+            Err(InvalidSyncFile(_)) => continue,
-                Err(e) => {
+            Err(e) => {
-                    warn!("Failed to read sync file {e}");
+                warn!("Failed to read sync file {e}");
-                    return None;
+                continue;
-                }
-            };
-
-            // Skip models belonging to different workspace
-            if model.workspace_id() != workspace.id.as_str() {
-                debug!("Skipping non-workspace file");
-                return None;
             }
+        };
 
-            let rel_path = Path::new(&dir_entry.file_name()).to_path_buf();
+        // Skip models belonging to different workspace
-            Some(FsCandidate {
+        if model.workspace_id() != workspace.id.as_str() {
-                rel_path,
+            debug!("Skipping non-workspace file");
-                model,
+            continue;
-                checksum,
+        }
-            })
+
+        let rel_path = Path::new(&dir_entry.file_name()).to_path_buf();
+        candidates.push(FsCandidate {
+            rel_path,
+            model,
+            checksum,
         })
-        .collect();
+    }
 
     Ok(candidates)
 }
@@ -363,7 +366,8 @@ async fn apply_sync_op<R: Runtime>(
             let rel_path = derive_model_filename(&model);
             let abs_path = derive_full_model_path(workspace, &model)?;
             let (content, checksum) = model.to_file_contents(&rel_path)?;
-            fs::write(&abs_path, content)?;
+            let mut f = File::create(&abs_path).await?;
+            f.write_all(&content).await?;
             SyncStateOp::Create {
                 model_id: model.id(),
                 checksum,
@@ -374,7 +378,8 @@ async fn apply_sync_op<R: Runtime>(
             let rel_path = derive_model_filename(&model);
             let abs_path = derive_full_model_path(workspace, &model)?;
             let (content, checksum) = model.to_file_contents(&rel_path)?;
-            fs::write(&abs_path, content)?;
+            let mut f = File::create(&abs_path).await?;
+            f.write_all(&content).await?;
             SyncStateOp::Update {
                 state: state.to_owned(),
                 checksum,
@@ -390,7 +395,7 @@ async fn apply_sync_op<R: Runtime>(
             },
             Some(fs_candidate) => {
                 let abs_path = derive_full_model_path(workspace, &fs_candidate.model)?;
-                fs::remove_file(&abs_path)?;
+                fs::remove_file(&abs_path).await?;
                 SyncStateOp::Delete {
                     state: state.to_owned(),
                 }

src-web/components/WorkspaceActionsDropdown.tsx

@@ -109,49 +109,51 @@ export const WorkspaceActionsDropdown = memo(function WorkspaceActionsDropdown({
             description: (
               <VStack space={3}>
                 <p>
-                  Some files in the directory have changed. Do you want to apply the updates to your
+                  {pluralizeCount('file', dbChanges.length)} in the directory have changed. Do you want to apply the updates to your
                   workspace?
                 </p>
-                <table className="w-full text-sm mb-auto min-w-full max-w-full divide-y divide-surface-highlight">
+                <div className="overflow-y-auto max-h-[10rem]">
-                  <thead>
+                  <table className="w-full text-sm mb-auto min-w-full max-w-full divide-y divide-surface-highlight">
-                    <tr>
+                    <thead>
-                      <th className="py-1 text-left">Name</th>
+                      <tr>
-                      <th className="py-1 text-right pl-4">Operation</th>
+                        <th className="py-1 text-left">Name</th>
-                    </tr>
+                        <th className="py-1 text-right pl-4">Operation</th>
-                  </thead>
+                      </tr>
-                  <tbody className="divide-y divide-surface-highlight">
+                    </thead>
-                    {dbChanges.map((op, i) => {
+                    <tbody className="divide-y divide-surface-highlight">
-                      let name = '';
+                      {dbChanges.map((op, i) => {
-                      let label = '';
+                        let name = '';
-                      let color = '';
+                        let label = '';
+                        let color = '';
 
-                      if (op.type === 'dbCreate') {
+                        if (op.type === 'dbCreate') {
-                        label = 'create';
+                          label = 'create';
-                        name = fallbackRequestName(op.fs.model);
+                          name = fallbackRequestName(op.fs.model);
-                        color = 'text-success';
+                          color = 'text-success';
-                      } else if (op.type === 'dbUpdate') {
+                        } else if (op.type === 'dbUpdate') {
-                        label = 'update';
+                          label = 'update';
-                        name = fallbackRequestName(op.fs.model);
+                          name = fallbackRequestName(op.fs.model);
-                        color = 'text-info';
+                          color = 'text-info';
-                      } else if (op.type === 'dbDelete') {
+                        } else if (op.type === 'dbDelete') {
-                        label = 'delete';
+                          label = 'delete';
-                        name = fallbackRequestName(op.model);
+                          name = fallbackRequestName(op.model);
-                        color = 'text-danger';
+                          color = 'text-danger';
-                      } else {
+                        } else {
-                        return null;
+                          return null;
-                      }
+                        }
 
-                      return (
+                        return (
-                        <tr key={i} className="text-text">
+                          <tr key={i} className="text-text">
-                          <td className="py-1">{name}</td>
+                            <td className="py-1">{name}</td>
-                          <td className="py-1 pl-4 text-right">
+                            <td className="py-1 pl-4 text-right">
-                            <InlineCode className={color}>{label}</InlineCode>
+                              <InlineCode className={color}>{label}</InlineCode>
-                          </td>
+                            </td>
-                        </tr>
+                          </tr>
-                      );
+                        );
-                    })}
+                      })}
-                  </tbody>
+                    </tbody>
-                </table>
+                  </table>
+                </div>
               </VStack>
             ),
           });