improvement(library): index speed

src/checksum.rs

@@ -1,11 +1,7 @@
-use sha2::{self, Digest, Sha256};
+use blake3::hash;
 
-pub fn generate_hash(content: impl AsRef<[u8]>) -> String {
-    let mut hasher = Sha256::new();
-    hasher.update(content);
-    let result = hasher.finalize();
+pub fn generate_hash(content: &[u8]) -> String {
+    let hash = hash(content);
 
-    let hex = hex::encode(result);
-
-    hex
+    hash.to_string()
 }

src/library.rs

@@ -1,5 +1,5 @@
-use rayon::prelude::*;
-use std::{collections::HashMap, fs, path::PathBuf, time::Instant};
+use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Instant};
+use tokio::{sync::Mutex, task::JoinSet};
 
 use r2d2::{Pool, PooledConnection};
 use r2d2_sqlite::SqliteConnectionManager;
@@ -56,7 +56,7 @@ impl Library for LibraryService {
     }
 }
 
-pub fn index_path(
+pub async fn index_path(
     path: PathBuf,
     db: PooledConnection<SqliteConnectionManager>,
     path_id: u64,
@@ -73,45 +73,37 @@ pub fn index_path(
         .filter_map(Result::ok)
         .collect();
 
-    let hashmaps: Vec<HashMap<String, TrackMetadata>> = entries
-        .into_par_iter()
-        .fold(
-            || HashMap::new(),
-            |mut acc: HashMap<String, TrackMetadata>, entry| {
-                if entry.file_type().is_file()
-                    && entry.path().extension().is_some_and(|ext| ext == "mp3")
-                {
-                    let file_path = entry.path();
-                    let content = fs::read(file_path).unwrap();
+    let tracks: Arc<Mutex<HashMap<String, TrackMetadata>>> = Arc::new(Mutex::new(HashMap::new()));
+    let mut set = JoinSet::new();
 
+    for entry in entries.into_iter() {
+        if entry.file_type().is_file() && entry.path().extension().is_some_and(|ext| ext == "mp3") {
+            let tracks = tracks.clone();
+            set.spawn(async move {
+                let file_path = entry.path();
+                if let Ok(content) = std::fs::read(file_path) {
                     let hash = generate_hash(&content);
 
                     let relative_path =
                         file_path.to_str().unwrap().to_string()[path_offset..].to_string();
 
                     if let Some(metadata) = extract_track_data(content, relative_path) {
-                        acc.insert(hash, metadata);
+                        tracks.lock().await.insert(hash, metadata);
                     }
                 }
-
-                acc
-            },
-        )
-        .collect();
-
-    let mut tracks = HashMap::<String, TrackMetadata>::new();
-
-    for tracks_chunk in hashmaps {
-        tracks.extend(tracks_chunk);
+            });
+        }
     }
 
+    set.join_all().await;
+
     let elapsed = now.elapsed();
 
     println!("indexing took {:.2?}", elapsed);
 
     let now = Instant::now();
 
-    insert_tracks(db, tracks, path_id)?;
+    insert_tracks(db, Arc::try_unwrap(tracks).unwrap().into_inner(), path_id)?;
 
     let elapsed = now.elapsed();
 

src/settings.rs

@@ -112,7 +112,7 @@ impl Settings for SettingsService {
             return Err(tonic::Status::not_found(""));
         };
 
-        let _ = index_path(library_path.path.into(), db, library_path.id);
+        let _ = index_path(library_path.path.into(), db, library_path.id).await;
 
         let response = RefreshPathResponse {};