Optimize sync performance by skipping npm install when package.json is unchanged

ast/src/repo.rs

@@ -21,20 +21,51 @@ pub async fn clone_repo(
     pat: Option<String>,
     commit: Option<&str>,
 ) -> Result<()> {
+    let mut package_json_changed = false;
+
     // check if the path exists
     if fs::metadata(path).is_ok() {
+        let check_pkg_json_env = std::env::var("CHECK_PACKAGE_JSON").unwrap_or_default();
+        let check_pkg_json = check_pkg_json_env == "true" || check_pkg_json_env == "1";
+
+        let mut old_package_json = String::new();
+        if check_pkg_json {
+            let package_json_path = format!("{}/package.json", path);
+            if let Ok(content) = fs::read_to_string(&package_json_path) {
+                old_package_json = content;
+            }
+        }
+
         // delete unless SKIP_RECLONE is set
         let skip_reclone_env = std::env::var("SKIP_RECLONE").unwrap_or_default();
         let skip_reclone = skip_reclone_env == "true" || skip_reclone_env == "1";
         if !skip_reclone {
             fs::remove_dir_all(path).ok();
             git_clone(url, path, username, pat, commit).await?;
+
+            if check_pkg_json && !old_package_json.is_empty() {
+                let package_json_path = format!("{}/package.json", path);
+                if let Ok(new_content) = fs::read_to_string(&package_json_path) {
+                    package_json_changed = old_package_json != new_content;
+                    if package_json_changed {
+                        info!("=> package.json changed, will run npm install");
+                    } else {
+                        info!("=> package.json unchanged, skipping npm install");
+                    }
+                }
+            }
         } else {
             info!("=> Skipping reclone for {:?}", path);
         }
     } else {
         git_clone(url, path, username, pat, commit).await?;
+        package_json_changed = true;
     }
+
+    if package_json_changed {
+        std::env::remove_var("CHECK_PACKAGE_JSON");
+    }
+
     Ok(())
 }
 
@@ -159,6 +190,12 @@ impl Repo {
         } else {
             revs.len() / urls.len()
         };
+
+        let is_sync = std::env::var("IS_SYNC").unwrap_or_default() == "true";
+        if is_sync {
+            std::env::set_var("CHECK_PACKAGE_JSON", "true");
+        }
+
         let mut repos: Vec<Repo> = Vec::new();
         for (i, url) in urls.iter().enumerate() {
             let gurl = GitUrl::parse(url)
@@ -297,6 +334,16 @@ impl Repo {
         })
     }
     fn run_cmd(cmd: &str, root: &str) -> Result<()> {
+        if cmd.starts_with("npm install") {
+            let check_pkg_json_env = std::env::var("CHECK_PACKAGE_JSON").unwrap_or_default();
+            let check_pkg_json = check_pkg_json_env == "true" || check_pkg_json_env == "1";
+
+            if check_pkg_json {
+                info!("Skipping npm install as package.json didn't change");
+                return Ok(());
+            }
+        }
+
         info!("Running cmd: {:?}", cmd);
         let mut arr = cmd.split(" ").collect::<Vec<&str>>();
         if arr.len() == 0 {

ast/src/utils.rs

@@ -100,6 +100,19 @@ pub fn get_use_lsp() -> bool {
     false
 }
 
+pub fn get_use_lsp_for_sync() -> bool {
+    println!("===-==> Getting use LSP for sync");
+    env::set_var("LSP_SKIP_POST_CLONE", "true");
+    env::set_var("CHECK_PACKAGE_JSON", "true");
+    env::set_var("IS_SYNC", "true");
+    delete_react_testing_node_modules().ok();
+    let lsp = env::var("USE_LSP").unwrap_or_else(|_| "false".to_string());
+    if lsp == "true" || lsp == "1" {
+        return true;
+    }
+    false
+}
+
 fn delete_react_testing_node_modules() -> std::io::Result<()> {
     let path = std::path::Path::new("src/testing/react/node_modules");
     if path.exists() {

lsp/src/language.rs

@@ -253,6 +253,13 @@ impl Language {
                 return Vec::new();
             }
         }
+
+        if let Ok(check_pkg_json) = std::env::var("CHECK_PACKAGE_JSON") {
+            if check_pkg_json == "true" || check_pkg_json == "1" {
+                return Vec::new();
+            }
+        }
+
         match self {
             Self::Rust => Vec::new(),
             Self::Go => Vec::new(),
@@ -270,6 +277,15 @@ impl Language {
         }
     }
 
+    pub fn npm_install_cmd(&self) -> Option<&'static str> {
+        match self {
+            Self::Typescript | Self::React => Some("npm install --force"),
+            Self::Angular => Some("npm install --force"),
+            Self::Svelte => Some("npm install --force"),
+            _ => None,
+        }
+    }
+
     pub fn test_id_regex(&self) -> Option<&'static str> {
         match self {
             Self::Typescript | Self::React => {

standalone/src/handlers.rs

@@ -74,7 +74,11 @@ pub async fn process(body: Json<ProcessBody>) -> Result<Json<ProcessResponse>> {
         )));
     }
     let (final_repo_path, final_repo_url, username, pat, _) = resolve_repo(&body)?;
-    let use_lsp = body.use_lsp;
+    let use_lsp = if body.use_lsp.unwrap_or_default() {
+        ast::utils::get_use_lsp_for_sync()
+    } else {
+        false
+    };
 
     let total_start = Instant::now();
 
@@ -129,7 +133,7 @@ pub async fn process(body: Json<ProcessBody>) -> Result<Json<ProcessResponse>> {
                 &current_hash,
                 &hash,
                 None,
-                use_lsp,
+                Some(use_lsp),
             )
             .await?
     } else {
@@ -141,7 +145,7 @@ pub async fn process(body: Json<ProcessBody>) -> Result<Json<ProcessResponse>> {
                 pat.clone(),
                 &current_hash,
                 None,
-                use_lsp,
+                Some(use_lsp),
             )
             .await?
     };
@@ -284,13 +288,17 @@ pub async fn ingest_async(
         );
     }
 
+    let mut ingest_body = body.0.clone();
+    if ingest_body.use_lsp.unwrap_or_default() {
+        ingest_body.use_lsp = Some(ast::utils::get_use_lsp());
+    }
+
     let state_clone = state.clone();
-    let body_clone = body.clone();
     let request_id_clone = request_id.clone();
 
     //run ingest as a background task
     tokio::spawn(async move {
-        let result = ingest(State(state_clone), body_clone).await;
+        let result = ingest(State(state_clone), Json(ingest_body)).await;
         let mut map = status_map.lock().await;
 
         match result {
@@ -332,12 +340,17 @@ pub async fn sync_async(
             },
         );
     }
-    let body_clone = body.clone();
+
+    let mut sync_body = body.0.clone();
+    if sync_body.use_lsp.unwrap_or_default() {
+        sync_body.use_lsp = Some(ast::utils::get_use_lsp_for_sync());
+    }
+
     let request_id_clone = request_id.clone();
 
     //run /sync as a background task
     tokio::spawn(async move {
-        let result = process(body_clone).await;
+        let result = process(Json(sync_body)).await;
         let mut map = status_map.lock().await;
 
         match result {