diff --git a/backend/src/lib/outbound/file_system.rs b/backend/src/lib/outbound/file_system.rs index 5a13e7f..55051c6 100644 --- a/backend/src/lib/outbound/file_system.rs +++ b/backend/src/lib/outbound/file_system.rs @@ -30,30 +30,46 @@ use crate::{ }, }; -const MAX_FILE_FETCH_BYTES: &str = "MAX_FILE_FETCH_BYTES"; +const MAX_FILE_FETCH_BYTES_KEY: &str = "MAX_FILE_FETCH_BYTES"; +const ZIP_READ_BUFFER_BYTES_KEY: &str = "ZIP_READ_BUFFER_BYTES"; #[derive(Debug, Clone)] pub struct FileSystemConfig { base_directory: String, max_file_fetch_bytes: u64, + zip_read_buffer_bytes: usize, } impl FileSystemConfig { - pub fn new(base_directory: String, max_file_fetch_bytes: u64) -> Self { + pub fn new( + base_directory: String, + max_file_fetch_bytes: u64, + zip_read_buffer_bytes: usize, + ) -> Self { Self { base_directory, max_file_fetch_bytes, + zip_read_buffer_bytes, } } pub fn from_env(serve_dir: String) -> anyhow::Result { // 268435456 bytes = 0.25GB - let max_file_fetch_bytes: u64 = match Config::load_env(MAX_FILE_FETCH_BYTES) { + let max_file_fetch_bytes: u64 = match Config::load_env(MAX_FILE_FETCH_BYTES_KEY) { Ok(value) => value.parse()?, Err(_) => 268435456, }; - Ok(Self::new(serve_dir, max_file_fetch_bytes)) + let zip_read_buffer_bytes: usize = match Config::load_env(ZIP_READ_BUFFER_BYTES_KEY) { + Ok(value) => value.parse()?, + Err(_) => 4096, + }; + + Ok(Self::new( + serve_dir, + max_file_fetch_bytes, + zip_read_buffer_bytes, + )) } } @@ -61,6 +77,7 @@ impl FileSystemConfig { pub struct FileSystem { base_directory: FilePath, max_file_fetch_bytes: u64, + zip_read_buffer_bytes: usize, } impl FileSystem { @@ -68,6 +85,7 @@ impl FileSystem { let file_system = Self { base_directory: FilePath::new(&config.base_directory)?, max_file_fetch_bytes: config.max_file_fetch_bytes, + zip_read_buffer_bytes: config.zip_read_buffer_bytes, }; Ok(file_system) @@ -227,7 +245,7 @@ impl FileSystem { std::sync::mpsc::channel::>(); let (tx, rx) = tokio::sync::mpsc::channel::>(1024); - tokio::task::spawn(create_zip(path, sync_tx)); + tokio::task::spawn(create_zip(path, sync_tx, self.zip_read_buffer_bytes)); tokio::task::spawn(async move { while let Ok(v) = sync_rx.recv() { let _ = tx.send(v).await; @@ -460,9 +478,15 @@ where Ok(files) } +/// Creates a ZIP archive from a directory +/// +/// * `path`: The directory's path +/// * `tx`: The sender for the new ZIP archive's bytes +/// * `buffer_size`: The size of the file read buffer. A large buffer increases both the speed and the memory usage async fn create_zip

( path: P, tx: std::sync::mpsc::Sender>, + buffer_size: usize, ) -> anyhow::Result<()> where P: AsRef, @@ -473,7 +497,7 @@ where .large_file(true) .unix_permissions(0o644); - let mut file_buf = Vec::new(); + let mut file_buf = vec![0; buffer_size]; let mut zip = zip::write::ZipWriter::new_stream(ChannelWriter(tx)); let entries = walk_dir(&path).await?; @@ -493,10 +517,11 @@ where zip.start_file(entry_str, options)?; let mut entry_file = tokio::fs::File::open(entry_path).await?; - entry_file.read_to_end(&mut file_buf).await?; - - zip.write_all(&file_buf)?; - file_buf.clear(); + while let Ok(len) = entry_file.read(&mut file_buf).await + && len > 0 + { + zip.write(&file_buf[..len])?; + } } zip.finish()?;