diff --git a/Cargo.lock b/Cargo.lock index 97d84af..531cb0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,6 +89,12 @@ dependencies = [ "syn", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" @@ -282,12 +288,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" [[package]] -name = "block-buffer" -version = "0.10.4" +name = "blake3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" dependencies = [ - "generic-array", + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", ] [[package]] @@ -401,6 +411,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -450,15 +466,6 @@ dependencies = [ "windows", ] -[[package]] -name = "cpufeatures" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" -dependencies = [ - "libc", -] - [[package]] name = "crc32fast" version = "1.4.2" @@ -499,32 +506,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "dasp_sample" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - [[package]] name = "dotenvy" version = "0.15.7" @@ -680,16 +667,6 @@ dependencies = [ "pin-utils", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.15" @@ -728,6 +705,7 @@ name = "groove" version = "0.1.0" dependencies = [ "axum", + "blake3", "dotenvy", "hex", "home", @@ -738,7 +716,6 @@ dependencies = [ "rayon", "rodio", "rusqlite", - "sha2", "symphonia", "tokio", "tokio-stream", @@ -1984,17 +1961,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "shlex" version = "1.3.0" @@ -2575,12 +2541,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - [[package]] name = "unicase" version = "2.7.0" diff --git a/Cargo.toml b/Cargo.toml index ba89c50..fdd70f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] axum = "0.7.9" +blake3 = "1.5.4" dotenvy = "0.15.7" hex = "0.4.3" home = "0.5.9" @@ -15,7 +16,6 @@ r2d2_sqlite = { version = "0.25.0", features = ["bundled"] } rayon = "1.10.0" rodio = "0.20.1" rusqlite = { version = "0.32.1", features = ["bundled"] } -sha2 = "0.10.8" symphonia = { version = "0.5.4", features = ["mp3"] } tokio = { version = "1.41.1", features = ["full"] } tokio-stream = "0.1.16" diff --git a/src/checksum.rs b/src/checksum.rs index 1416764..0c10900 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -1,11 +1,7 @@ -use sha2::{self, Digest, Sha256}; +use blake3::hash; -pub fn generate_hash(content: impl AsRef<[u8]>) -> String { - let mut hasher = Sha256::new(); - hasher.update(content); - let result = hasher.finalize(); +pub fn generate_hash(content: &[u8]) -> String { + let hash = hash(content); - let hex = hex::encode(result); - - hex + hash.to_string() } diff --git a/src/library.rs b/src/library.rs index 659ac29..72b14d7 100644 --- a/src/library.rs +++ b/src/library.rs @@ -1,5 +1,5 @@ -use rayon::prelude::*; -use std::{collections::HashMap, fs, path::PathBuf, time::Instant}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Instant}; +use tokio::{sync::Mutex, task::JoinSet}; use r2d2::{Pool, PooledConnection}; use r2d2_sqlite::SqliteConnectionManager; @@ -56,7 +56,7 @@ impl Library for LibraryService { } } -pub fn index_path( +pub async fn index_path( path: PathBuf, db: PooledConnection, path_id: u64, @@ -73,45 +73,37 @@ pub fn index_path( .filter_map(Result::ok) .collect(); - let hashmaps: Vec> = entries - .into_par_iter() - .fold( - || HashMap::new(), - |mut acc: HashMap, entry| { - if entry.file_type().is_file() - && entry.path().extension().is_some_and(|ext| ext == "mp3") - { - let file_path = entry.path(); - let content = fs::read(file_path).unwrap(); + let tracks: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let mut set = JoinSet::new(); + for entry in entries.into_iter() { + if entry.file_type().is_file() && entry.path().extension().is_some_and(|ext| ext == "mp3") { + let tracks = tracks.clone(); + set.spawn(async move { + let file_path = entry.path(); + if let Ok(content) = std::fs::read(file_path) { let hash = generate_hash(&content); let relative_path = file_path.to_str().unwrap().to_string()[path_offset..].to_string(); if let Some(metadata) = extract_track_data(content, relative_path) { - acc.insert(hash, metadata); + tracks.lock().await.insert(hash, metadata); } } - - acc - }, - ) - .collect(); - - let mut tracks = HashMap::::new(); - - for tracks_chunk in hashmaps { - tracks.extend(tracks_chunk); + }); + } } + set.join_all().await; + let elapsed = now.elapsed(); println!("indexing took {:.2?}", elapsed); let now = Instant::now(); - insert_tracks(db, tracks, path_id)?; + insert_tracks(db, Arc::try_unwrap(tracks).unwrap().into_inner(), path_id)?; let elapsed = now.elapsed(); diff --git a/src/settings.rs b/src/settings.rs index f704663..3e5aeff 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -112,7 +112,7 @@ impl Settings for SettingsService { return Err(tonic::Status::not_found("")); }; - let _ = index_path(library_path.path.into(), db, library_path.id); + let _ = index_path(library_path.path.into(), db, library_path.id).await; let response = RefreshPathResponse {};