409/groove
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improvement(library): index speed

Browse Source
This commit is contained in:
Shawn
2024-11-27 03:57:05 +01:00
parent 22e6fe31b3
commit 94b8b15f80
5 changed files with 44 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
Cargo.lock generated
View File

@@ -89,6 +89,12 @@ dependencies = [
"syn",
]
[[package]]
name = "arrayref"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
[[package]]
name = "arrayvec"
version = "0.7.6"
@@ -282,12 +288,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2"
[[package]]
name = "block-buffer"
version = "0.10.4"
name = "blake3"
version = "1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7"
dependencies = [
"generic-array",
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
]
[[package]]
@@ -401,6 +411,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "constant_time_eq"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
@@ -450,15 +466,6 @@ dependencies = [
"windows",
]
[[package]]
name = "cpufeatures"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3"
dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.4.2"
@@ -499,32 +506,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "dasp_sample"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "dotenvy"
version = "0.15.7"
@@ -680,16 +667,6 @@ dependencies = [
"pin-utils",
]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.15"
@@ -728,6 +705,7 @@ name = "groove"
version = "0.1.0"
dependencies = [
"axum",
"blake3",
"dotenvy",
"hex",
"home",
@@ -738,7 +716,6 @@ dependencies = [
"rayon",
"rodio",
"rusqlite",
"sha2",
"symphonia",
"tokio",
"tokio-stream",
@@ -1984,17 +1961,6 @@ dependencies = [
"serde",
]
[[package]]
name = "sha2"
version = "0.10.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "shlex"
version = "1.3.0"
@@ -2575,12 +2541,6 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "typenum"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "unicase"
version = "2.7.0"

2
Cargo.toml
View File

@@ -5,6 +5,7 @@ edition = "2021"
[dependencies]
axum = "0.7.9"
blake3 = "1.5.4"
dotenvy = "0.15.7"
hex = "0.4.3"
home = "0.5.9"
@@ -15,7 +16,6 @@ r2d2_sqlite = { version = "0.25.0", features = ["bundled"] }
rayon = "1.10.0"
rodio = "0.20.1"
rusqlite = { version = "0.32.1", features = ["bundled"] }
sha2 = "0.10.8"
symphonia = { version = "0.5.4", features = ["mp3"] }
tokio = { version = "1.41.1", features = ["full"] }
tokio-stream = "0.1.16"

12
src/checksum.rs
View File

@@ -1,11 +1,7 @@
use sha2::{self, Digest, Sha256};
use blake3::hash;
pub fn generate_hash(content: impl AsRef<[u8]>) -> String {
let mut hasher = Sha256::new();
hasher.update(content);
let result = hasher.finalize();
pub fn generate_hash(content: &[u8]) -> String {
let hash = hash(content);
let hex = hex::encode(result);
hex
hash.to_string()
}

42
src/library.rs
View File

@@ -1,5 +1,5 @@
use rayon::prelude::*;
use std::{collections::HashMap, fs, path::PathBuf, time::Instant};
use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Instant};
use tokio::{sync::Mutex, task::JoinSet};
use r2d2::{Pool, PooledConnection};
use r2d2_sqlite::SqliteConnectionManager;
@@ -56,7 +56,7 @@ impl Library for LibraryService {
}
}
pub fn index_path(
pub async fn index_path(
path: PathBuf,
db: PooledConnection<SqliteConnectionManager>,
path_id: u64,
@@ -73,37 +73,29 @@ pub fn index_path(
.filter_map(Result::ok)
.collect();
let hashmaps: Vec<HashMap<String, TrackMetadata>> = entries
.into_par_iter()
.fold(
|| HashMap::new(),
|mut acc: HashMap<String, TrackMetadata>, entry| {
if entry.file_type().is_file()
&& entry.path().extension().is_some_and(|ext| ext == "mp3")
{
let file_path = entry.path();
let content = fs::read(file_path).unwrap();
let tracks: Arc<Mutex<HashMap<String, TrackMetadata>>> = Arc::new(Mutex::new(HashMap::new()));
let mut set = JoinSet::new();
for entry in entries.into_iter() {
if entry.file_type().is_file() && entry.path().extension().is_some_and(|ext| ext == "mp3") {
let tracks = tracks.clone();
set.spawn(async move {
let file_path = entry.path();
if let Ok(content) = std::fs::read(file_path) {
let hash = generate_hash(&content);
let relative_path =
file_path.to_str().unwrap().to_string()[path_offset..].to_string();
if let Some(metadata) = extract_track_data(content, relative_path) {
acc.insert(hash, metadata);
tracks.lock().await.insert(hash, metadata);
}
}
});
}
}
acc
},
)
.collect();
let mut tracks = HashMap::<String, TrackMetadata>::new();
for tracks_chunk in hashmaps {
tracks.extend(tracks_chunk);
}
set.join_all().await;
let elapsed = now.elapsed();
@@ -111,7 +103,7 @@ pub fn index_path(
let now = Instant::now();
insert_tracks(db, tracks, path_id)?;
insert_tracks(db, Arc::try_unwrap(tracks).unwrap().into_inner(), path_id)?;
let elapsed = now.elapsed();

2
src/settings.rs
View File

@@ -112,7 +112,7 @@ impl Settings for SettingsService {
return Err(tonic::Status::not_found(""));
};
let _ = index_path(library_path.path.into(), db, library_path.id);
let _ = index_path(library_path.path.into(), db, library_path.id).await;
let response = RefreshPathResponse {};