improvement(library): index speed

Cargo.lock

@@ -89,6 +89,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "arrayref"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.6"
@@ -282,12 +288,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2"
 
 [[package]]
-name = "block-buffer"
+name = "blake3"
-version = "0.10.4"
+version = "1.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7"
 dependencies = [
- "generic-array",
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
 ]
 
 [[package]]
@@ -401,6 +411,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "constant_time_eq"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.7"
@@ -450,15 +466,6 @@ dependencies = [
  "windows",
 ]
 
-[[package]]
-name = "cpufeatures"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "crc32fast"
 version = "1.4.2"
@@ -499,32 +506,12 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
 
-[[package]]
-name = "crypto-common"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
 [[package]]
 name = "dasp_sample"
 version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
 
-[[package]]
-name = "digest"
-version = "0.10.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
-dependencies = [
- "block-buffer",
- "crypto-common",
-]
-
 [[package]]
 name = "dotenvy"
 version = "0.15.7"
@@ -680,16 +667,6 @@ dependencies = [
  "pin-utils",
 ]
 
-[[package]]
-name = "generic-array"
-version = "0.14.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
-dependencies = [
- "typenum",
- "version_check",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.15"
@@ -728,6 +705,7 @@ name = "groove"
 version = "0.1.0"
 dependencies = [
  "axum",
+ "blake3",
  "dotenvy",
  "hex",
  "home",
@@ -738,7 +716,6 @@ dependencies = [
  "rayon",
  "rodio",
  "rusqlite",
- "sha2",
  "symphonia",
  "tokio",
  "tokio-stream",
@@ -1984,17 +1961,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "sha2"
-version = "0.10.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
-
 [[package]]
 name = "shlex"
 version = "1.3.0"
@@ -2575,12 +2541,6 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
-[[package]]
-name = "typenum"
-version = "1.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
-
 [[package]]
 name = "unicase"
 version = "2.7.0"

Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 axum = "0.7.9"
+blake3 = "1.5.4"
 dotenvy = "0.15.7"
 hex = "0.4.3"
 home = "0.5.9"
@@ -15,7 +16,6 @@ r2d2_sqlite = { version = "0.25.0", features = ["bundled"] }
 rayon = "1.10.0"
 rodio = "0.20.1"
 rusqlite = { version = "0.32.1", features = ["bundled"] }
-sha2 = "0.10.8"
 symphonia = { version = "0.5.4", features = ["mp3"] }
 tokio = { version = "1.41.1", features = ["full"] }
 tokio-stream = "0.1.16"

src/checksum.rs

@@ -1,11 +1,7 @@
-use sha2::{self, Digest, Sha256};
+use blake3::hash;
 
-pub fn generate_hash(content: impl AsRef<[u8]>) -> String {
+pub fn generate_hash(content: &[u8]) -> String {
-    let mut hasher = Sha256::new();
+    let hash = hash(content);
-    hasher.update(content);
-    let result = hasher.finalize();
 
-    let hex = hex::encode(result);
+    hash.to_string()
-
-    hex
 }

src/library.rs

@@ -1,5 +1,5 @@
-use rayon::prelude::*;
+use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Instant};
-use std::{collections::HashMap, fs, path::PathBuf, time::Instant};
+use tokio::{sync::Mutex, task::JoinSet};
 
 use r2d2::{Pool, PooledConnection};
 use r2d2_sqlite::SqliteConnectionManager;
@@ -56,7 +56,7 @@ impl Library for LibraryService {
     }
 }
 
-pub fn index_path(
+pub async fn index_path(
     path: PathBuf,
     db: PooledConnection<SqliteConnectionManager>,
     path_id: u64,
@@ -73,45 +73,37 @@ pub fn index_path(
         .filter_map(Result::ok)
         .collect();
 
-    let hashmaps: Vec<HashMap<String, TrackMetadata>> = entries
+    let tracks: Arc<Mutex<HashMap<String, TrackMetadata>>> = Arc::new(Mutex::new(HashMap::new()));
-        .into_par_iter()
+    let mut set = JoinSet::new();
-        .fold(
-            || HashMap::new(),
-            |mut acc: HashMap<String, TrackMetadata>, entry| {
-                if entry.file_type().is_file()
-                    && entry.path().extension().is_some_and(|ext| ext == "mp3")
-                {
-                    let file_path = entry.path();
-                    let content = fs::read(file_path).unwrap();
 
+    for entry in entries.into_iter() {
+        if entry.file_type().is_file() && entry.path().extension().is_some_and(|ext| ext == "mp3") {
+            let tracks = tracks.clone();
+            set.spawn(async move {
+                let file_path = entry.path();
+                if let Ok(content) = std::fs::read(file_path) {
                     let hash = generate_hash(&content);
 
                     let relative_path =
                         file_path.to_str().unwrap().to_string()[path_offset..].to_string();
 
                     if let Some(metadata) = extract_track_data(content, relative_path) {
-                        acc.insert(hash, metadata);
+                        tracks.lock().await.insert(hash, metadata);
                     }
                 }
-
+            });
-                acc
+        }
-            },
-        )
-        .collect();
-
-    let mut tracks = HashMap::<String, TrackMetadata>::new();
-
-    for tracks_chunk in hashmaps {
-        tracks.extend(tracks_chunk);
     }
 
+    set.join_all().await;
+
     let elapsed = now.elapsed();
 
     println!("indexing took {:.2?}", elapsed);
 
     let now = Instant::now();
 
-    insert_tracks(db, tracks, path_id)?;
+    insert_tracks(db, Arc::try_unwrap(tracks).unwrap().into_inner(), path_id)?;
 
     let elapsed = now.elapsed();
 

src/settings.rs

@@ -112,7 +112,7 @@ impl Settings for SettingsService {
             return Err(tonic::Status::not_found(""));
         };
 
-        let _ = index_path(library_path.path.into(), db, library_path.id);
+        let _ = index_path(library_path.path.into(), db, library_path.id).await;
 
         let response = RefreshPathResponse {};