409/warren
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve zip downloads

Browse Source
This commit is contained in:
Shawn
2025-08-30 02:42:38 +02:00
parent 9b3f4a5fe6
commit 702f16f199
5 changed files with 48 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
backend/src/lib/outbound/file_system.rs
View File

@@ -225,17 +225,16 @@ impl FileSystem {
let (sync_tx, sync_rx) =
std::sync::mpsc::channel::<Result<bytes::Bytes, std::io::Error>>();
let (tx, rx) =
tokio::sync::mpsc::channel::<Result<bytes::Bytes, std::io::Error>>(65536);
let (tx, rx) = tokio::sync::mpsc::channel::<Result<bytes::Bytes, std::io::Error>>(1024);
tokio::task::spawn(create_zip(path, sync_tx));
tokio::task::spawn(async move {
while let Ok(v) = sync_rx.recv() {
let _ = tx.send(v).await;
}
});
tokio::task::spawn(create_zip(path, sync_tx));
let stream = FileStream::new(ReceiverStream::new(rx));
let stream = FileStream::new(FileType::Directory, ReceiverStream::new(rx));
return Ok(stream);
}
@@ -246,7 +245,7 @@ impl FileSystem {
bail!("File size exceeds configured limit");
}
let stream = FileStream::new(ReaderStream::new(file));
let stream = FileStream::new(FileType::File, ReaderStream::new(file));
Ok(stream)
}
@@ -469,13 +468,17 @@ where
P: AsRef<Path>,
{
let options = zip::write::SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Deflated)
.compression_method(zip::CompressionMethod::Stored)
.compression_level(None)
.large_file(true)
.unix_permissions(0o644);
let mut file_buf = Vec::new();
let mut zip = zip::write::ZipWriter::new_stream(ChannelWriter(tx));
for entry_path_buf in walk_dir(&path).await? {
let entries = walk_dir(&path).await?;
for entry_path_buf in entries {
let entry_path = entry_path_buf.as_path();
let entry_str = entry_path
.strip_prefix(&path)?
@@ -489,12 +492,14 @@ where
zip.start_file(entry_str, options)?;
let mut entry_file = tokio::fs::File::open(entry_path).await?;
entry_file.read_to_end(&mut file_buf).await?;
zip.write_all(&file_buf)?;
file_buf.clear();
}
drop(zip.finish()?);
zip.finish()?;
Ok(())
}
@@ -504,7 +509,7 @@ struct ChannelWriter(std::sync::mpsc::Sender<Result<bytes::Bytes, std::io::Error
impl Write for ChannelWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let len = buf.len();
let data = bytes::Bytes::copy_from_slice(&buf[..(len)]);
let data = bytes::Bytes::copy_from_slice(buf);
self.0
.send(Ok(data))