409/warren
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

basic selection + download multiple files with selection

Browse Source
This commit is contained in:
Shawn
2025-09-02 18:08:13 +02:00
parent be46f92ddf
commit e2085c1baa
22 changed files with 516 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
backend/src/lib/domain/warren/models/file/requests/cat.rs
View File

@@ -1,29 +1,29 @@
use thiserror::Error;
use crate::domain::warren::models::file::AbsoluteFilePath;
use super::AbsoluteFilePathList;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct CatRequest {
path: AbsoluteFilePath,
paths: AbsoluteFilePathList,
}
impl CatRequest {
pub fn new(path: AbsoluteFilePath) -> Self {
Self { path }
pub fn new(paths: AbsoluteFilePathList) -> Self {
Self { paths }
}
pub fn path(&self) -> &AbsoluteFilePath {
&self.path
pub fn paths(&self) -> &AbsoluteFilePathList {
&self.paths
}
pub fn into_path(self) -> AbsoluteFilePath {
self.path
pub fn into_paths(self) -> AbsoluteFilePathList {
self.paths
}
}
#[derive(Debug, Error)]
pub enum CatError {
#[error("The file does not exist")]
#[error("A file does not exist")]
NotFound,
#[error(transparent)]
Unknown(#[from] anyhow::Error),

36
backend/src/lib/domain/warren/models/file/requests/mod.rs
View File

@@ -16,4 +16,40 @@ pub use mv::*;
pub use rm::*;
pub use save::*;
pub use stat::*;
use thiserror::Error;
pub use touch::*;
use super::AbsoluteFilePath;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct AbsoluteFilePathList(Vec<AbsoluteFilePath>);
impl From<AbsoluteFilePathList> for Vec<AbsoluteFilePath> {
fn from(value: AbsoluteFilePathList) -> Self {
value.0
}
}
impl AbsoluteFilePathList {
pub fn new(paths: Vec<AbsoluteFilePath>) -> Result<Self, AbsoluteFilePathListError> {
if paths.is_empty() {
return Err(AbsoluteFilePathListError::Empty);
}
Ok(Self(paths))
}
pub fn paths(&self) -> &Vec<AbsoluteFilePath> {
&self.0
}
pub fn paths_mut(&mut self) -> &mut Vec<AbsoluteFilePath> {
&mut self.0
}
}
#[derive(Debug, Error)]
pub enum AbsoluteFilePathListError {
#[error("A list must not be empty")]
Empty,
}

42
backend/src/lib/domain/warren/models/share/requests/cat.rs
View File

@@ -2,7 +2,7 @@ use thiserror::Error;
use uuid::Uuid;
use crate::domain::warren::models::{
file::{AbsoluteFilePath, CatRequest, FileStream},
file::{AbsoluteFilePathList, CatRequest, FileStream},
share::{Share, SharePassword},
warren::{FetchWarrenError, Warren, WarrenCatError, WarrenCatRequest},
};
@@ -12,16 +12,16 @@ use super::{VerifySharePasswordError, VerifySharePasswordRequest};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ShareCatRequest {
share_id: Uuid,
path: AbsoluteFilePath,
base: CatRequest,
password: Option<SharePassword>,
}
impl ShareCatRequest {
pub fn new(share_id: Uuid, path: AbsoluteFilePath, password: Option<SharePassword>) -> Self {
pub fn new(share_id: Uuid, password: Option<SharePassword>, base: CatRequest) -> Self {
Self {
share_id,
path,
password,
base,
}
}
@@ -29,18 +29,23 @@ impl ShareCatRequest {
&self.share_id
}
pub fn path(&self) -> &AbsoluteFilePath {
&self.path
}
pub fn password(&self) -> Option<&SharePassword> {
self.password.as_ref()
}
pub fn build_warren_cat_request(self, share: &Share, warren: &Warren) -> WarrenCatRequest {
let path = share.path().clone().join(&self.path.to_relative());
pub fn base(&self) -> &CatRequest {
&self.base
}
WarrenCatRequest::new(*warren.id(), CatRequest::new(path))
pub fn build_warren_cat_request(self, share: &Share, warren: &Warren) -> WarrenCatRequest {
let mut paths = self.base.into_paths();
paths
.paths_mut()
.into_iter()
.for_each(|path| *path = share.path.clone().join(&path.clone().to_relative()));
WarrenCatRequest::new(*warren.id(), CatRequest::new(paths))
}
}
@@ -53,16 +58,21 @@ impl From<&ShareCatRequest> for VerifySharePasswordRequest {
pub struct ShareCatResponse {
share: Share,
warren: Warren,
path: AbsoluteFilePath,
paths: AbsoluteFilePathList,
stream: FileStream,
}
impl ShareCatResponse {
pub fn new(share: Share, warren: Warren, path: AbsoluteFilePath, stream: FileStream) -> Self {
pub fn new(
share: Share,
warren: Warren,
paths: AbsoluteFilePathList,
stream: FileStream,
) -> Self {
Self {
share,
warren,
path,
paths,
stream,
}
}
@@ -75,8 +85,8 @@ impl ShareCatResponse {
&self.warren
}
pub fn path(&self) -> &AbsoluteFilePath {
&self.path
pub fn paths(&self) -> &AbsoluteFilePathList {
&self.paths
}
pub fn stream(&self) -> &FileStream {

12
backend/src/lib/domain/warren/models/warren/requests.rs
View File

@@ -597,12 +597,14 @@ impl WarrenCatRequest {
}
pub fn build_fs_request(self, warren: &Warren) -> CatRequest {
let path = warren
.path()
.clone()
.join(&self.base.into_path().to_relative());
let mut paths = self.base.into_paths();
CatRequest::new(path)
paths
.paths_mut()
.into_iter()
.for_each(|path| *path = warren.path.clone().join(&path.clone().to_relative()));
CatRequest::new(paths)
}
}

8
backend/src/lib/domain/warren/ports/notifier.rs
View File

@@ -2,7 +2,7 @@ use uuid::Uuid;
use crate::domain::warren::models::{
auth_session::requests::FetchAuthSessionResponse,
file::{AbsoluteFilePath, LsResponse},
file::{AbsoluteFilePath, AbsoluteFilePathList, LsResponse},
share::{
CreateShareResponse, DeleteShareResponse, GetShareResponse, ListSharesResponse,
ShareCatResponse, ShareLsResponse,
@@ -28,7 +28,7 @@ pub trait WarrenNotifier: Clone + Send + Sync + 'static {
fn warren_cat(
&self,
warren: &Warren,
path: &AbsoluteFilePath,
path: &AbsoluteFilePathList,
) -> impl Future<Output = ()> + Send;
fn warren_mkdir(&self, response: &WarrenMkdirResponse) -> impl Future<Output = ()> + Send;
fn warren_rm(&self, response: &WarrenRmResponse) -> impl Future<Output = ()> + Send;
@@ -68,7 +68,7 @@ pub trait WarrenNotifier: Clone + Send + Sync + 'static {
pub trait FileSystemNotifier: Clone + Send + Sync + 'static {
fn ls(&self, response: &LsResponse) -> impl Future<Output = ()> + Send;
fn cat(&self, path: &AbsoluteFilePath) -> impl Future<Output = ()> + Send;
fn cat(&self, path: &AbsoluteFilePathList) -> impl Future<Output = ()> + Send;
fn mkdir(&self, path: &AbsoluteFilePath) -> impl Future<Output = ()> + Send;
fn rm(&self, path: &AbsoluteFilePath) -> impl Future<Output = ()> + Send;
fn mv(
@@ -163,7 +163,7 @@ pub trait AuthNotifier: Clone + Send + Sync + 'static {
&self,
user: &User,
warren_id: &Uuid,
path: &AbsoluteFilePath,
paths: &AbsoluteFilePathList,
) -> impl Future<Output = ()> + Send;
fn auth_warren_mkdir(
&self,

4
backend/src/lib/domain/warren/service/auth.rs
View File

@@ -704,7 +704,7 @@ where
return Err(AuthError::InsufficientPermissions);
}
let path = request.base().path().clone();
let paths = request.base().paths().clone();
let result = warren_service
.warren_cat(request)
@@ -714,7 +714,7 @@ where
if let Ok(_stream) = result.as_ref() {
self.metrics.record_auth_warren_cat_success().await;
self.notifier
.auth_warren_cat(&user, user_warren.warren_id(), &path)
.auth_warren_cat(&user, user_warren.warren_id(), &paths)
.await;
} else {
self.metrics.record_auth_warren_cat_failure().await;

4
backend/src/lib/domain/warren/service/file_system.rs
View File

@@ -54,12 +54,12 @@ where
}
async fn cat(&self, request: CatRequest) -> Result<FileStream, CatError> {
let path = request.path().clone();
let paths = request.paths().clone();
let result = self.repository.cat(request).await;
if result.is_ok() {
self.metrics.record_cat_success().await;
self.notifier.cat(&path).await;
self.notifier.cat(&paths).await;
} else {
self.metrics.record_cat_failure().await;
}

8
backend/src/lib/domain/warren/service/warren.rs
View File

@@ -157,14 +157,14 @@ where
async fn warren_cat(&self, request: WarrenCatRequest) -> Result<FileStream, WarrenCatError> {
let warren = self.repository.fetch_warren((&request).into()).await?;
let path = request.base().path().clone();
let paths = request.base().paths().clone();
let cat_request = request.build_fs_request(&warren);
let result = self.fs_service.cat(cat_request).await.map_err(Into::into);
if result.is_ok() {
self.metrics.record_warren_cat_success().await;
self.notifier.warren_cat(&warren, &path).await;
self.notifier.warren_cat(&warren, &paths).await;
} else {
self.metrics.record_warren_cat_failure().await;
}
@@ -517,7 +517,7 @@ where
}
};
let path = request.path().clone();
let paths = request.base().paths().clone();
let stream = match self
.warren_cat(request.build_warren_cat_request(&share, &warren))
@@ -530,7 +530,7 @@ where
}
};
let response = ShareCatResponse::new(share, warren, path, stream);
let response = ShareCatResponse::new(share, warren, paths, stream);
self.metrics.record_warren_share_cat_success().await;
self.notifier.warren_share_cat(&response).await;

28
backend/src/lib/inbound/http/handlers/warrens/cat_share.rs
View File

@@ -9,7 +9,10 @@ use uuid::Uuid;
use crate::{
domain::warren::{
models::{
file::{AbsoluteFilePathError, FilePath, FilePathError, FileStream},
file::{
AbsoluteFilePath, AbsoluteFilePathError, AbsoluteFilePathList,
AbsoluteFilePathListError, CatRequest, FilePath, FilePathError, FileStream,
},
share::{ShareCatRequest, SharePassword, SharePasswordError},
},
ports::{AuthService, WarrenService},
@@ -24,6 +27,8 @@ enum ParseShareCatHttpRequestError {
#[error(transparent)]
AbsoluteFilePath(#[from] AbsoluteFilePathError),
#[error(transparent)]
PathList(#[from] AbsoluteFilePathListError),
#[error(transparent)]
Password(#[from] SharePasswordError),
}
@@ -38,6 +43,11 @@ impl From<ParseShareCatHttpRequestError> for ApiError {
Self::BadRequest("The path must be absolute".to_string())
}
},
ParseShareCatHttpRequestError::PathList(err) => match err {
AbsoluteFilePathListError::Empty => {
Self::BadRequest("You must provide at least 1 path".to_string())
}
},
ParseShareCatHttpRequestError::Password(err) => Self::BadRequest(
match err {
SharePasswordError::Empty => "The provided password is empty",
@@ -56,7 +66,7 @@ impl From<ParseShareCatHttpRequestError> for ApiError {
#[serde(rename_all = "camelCase")]
pub(super) struct ShareCatHttpRequestBody {
share_id: Uuid,
path: String,
paths: String,
}
impl ShareCatHttpRequestBody {
@@ -64,9 +74,19 @@ impl ShareCatHttpRequestBody {
self,
password: Option<SharePassword>,
) -> Result<ShareCatRequest, ParseShareCatHttpRequestError> {
let path = FilePath::new(&self.path)?.try_into()?;
let mut paths = Vec::<AbsoluteFilePath>::new();
Ok(ShareCatRequest::new(self.share_id, path, password))
for path in self.paths.split(':') {
paths.push(FilePath::new(path)?.try_into()?);
}
let path_list = AbsoluteFilePathList::new(paths)?;
Ok(ShareCatRequest::new(
self.share_id,
password,
CatRequest::new(path_list),
))
}
}

29
backend/src/lib/inbound/http/handlers/warrens/warren_cat.rs
View File

@@ -10,7 +10,10 @@ use crate::{
domain::warren::{
models::{
auth_session::AuthRequest,
file::{AbsoluteFilePathError, CatRequest, FilePath, FilePathError},
file::{
AbsoluteFilePath, AbsoluteFilePathError, AbsoluteFilePathList,
AbsoluteFilePathListError, CatRequest, FilePath, FilePathError,
},
warren::WarrenCatRequest,
},
ports::{AuthService, WarrenService},
@@ -22,15 +25,17 @@ use crate::{
#[serde(rename_all = "camelCase")]
pub(super) struct WarrenCatHttpRequestBody {
warren_id: Uuid,
path: String,
paths: String,
}
#[derive(Debug, Clone, Error)]
#[derive(Debug, Error)]
pub enum ParseWarrenCatHttpRequestError {
#[error(transparent)]
FilePath(#[from] FilePathError),
#[error(transparent)]
AbsoluteFilePath(#[from] AbsoluteFilePathError),
#[error(transparent)]
PathList(#[from] AbsoluteFilePathListError),
}
impl From<ParseWarrenCatHttpRequestError> for ApiError {
@@ -46,15 +51,29 @@ impl From<ParseWarrenCatHttpRequestError> for ApiError {
ApiError::BadRequest("The file path must be absolute".to_string())
}
},
ParseWarrenCatHttpRequestError::PathList(err) => match err {
AbsoluteFilePathListError::Empty => {
Self::BadRequest("You must provide at least 1 path".to_string())
}
},
}
}
}
impl WarrenCatHttpRequestBody {
fn try_into_domain(self) -> Result<WarrenCatRequest, ParseWarrenCatHttpRequestError> {
let path = FilePath::new(&self.path)?.try_into()?;
let mut paths = Vec::<AbsoluteFilePath>::new();
Ok(WarrenCatRequest::new(self.warren_id, CatRequest::new(path)))
for path in self.paths.split(':') {
paths.push(FilePath::new(path)?.try_into()?);
}
let path_list = AbsoluteFilePathList::new(paths)?;
Ok(WarrenCatRequest::new(
self.warren_id,
CatRequest::new(path_list),
))
}
}

172
backend/src/lib/outbound/file_system.rs
View File

@@ -2,6 +2,7 @@ use anyhow::{Context as _, anyhow, bail};
use futures_util::TryStreamExt;
use rustix::fs::{Statx, statx};
use std::{
collections::HashSet,
io::Write,
os::unix::fs::MetadataExt,
path::{Path, PathBuf},
@@ -226,46 +227,79 @@ impl FileSystem {
Ok(paths)
}
async fn cat(&self, path: &AbsoluteFilePath) -> anyhow::Result<FileStream> {
let path = self.get_target_path(path);
async fn cat(&self, paths: &Vec<AbsoluteFilePath>) -> anyhow::Result<FileStream> {
let paths: Vec<FilePath> = paths
.into_iter()
.map(|path| self.get_target_path(path))
.collect();
let file = fs::OpenOptions::new()
.create(false)
.write(false)
.read(true)
.open(&path)
.await?;
let metadata = file.metadata().await?;
if metadata.is_dir() {
drop(file);
let path_request = PathRequest::from_paths(paths)?;
fn build_zip_stream(
prefix: String,
paths: Vec<FilePath>,
buffer_size: usize,
) -> FileStream {
let (sync_tx, sync_rx) =
std::sync::mpsc::channel::<Result<bytes::Bytes, std::io::Error>>();
let (tx, rx) = tokio::sync::mpsc::channel::<Result<bytes::Bytes, std::io::Error>>(1024);
tokio::task::spawn(create_zip(path, sync_tx, self.zip_read_buffer_bytes));
tokio::task::spawn(create_zip(
prefix,
paths
.into_iter()
.map(|p| PathBuf::from(p.as_str()))
.collect(),
sync_tx,
buffer_size,
));
tokio::task::spawn(async move {
while let Ok(v) = sync_rx.recv() {
let _ = tx.send(v).await;
}
});
let stream = FileStream::new(FileType::Directory, ReceiverStream::new(rx));
return Ok(stream);
FileStream::new(FileType::Directory, ReceiverStream::new(rx))
}
let file_size = metadata.size();
match path_request {
PathRequest::Single(file_path) => {
let file = fs::OpenOptions::new()
.create(false)
.write(false)
.read(true)
.open(&file_path)
.await?;
if file_size > self.max_file_fetch_bytes {
bail!("File size exceeds configured limit");
let metadata = file.metadata().await?;
if metadata.is_dir() {
drop(file);
return Ok(build_zip_stream(
file_path.to_string(),
vec![file_path],
self.zip_read_buffer_bytes,
));
}
if metadata.size() > self.max_file_fetch_bytes {
bail!("File size exceeds configured limit");
}
let stream = FileStream::new(FileType::File, ReaderStream::new(file));
Ok(stream)
}
PathRequest::Multiple {
lowest_common_prefix,
paths,
} => Ok(build_zip_stream(
lowest_common_prefix,
paths,
self.zip_read_buffer_bytes,
)),
}
let stream = FileStream::new(FileType::File, ReaderStream::new(file));
Ok(stream)
}
async fn mv(&self, path: &AbsoluteFilePath, target_path: &AbsoluteFilePath) -> io::Result<()> {
@@ -364,9 +398,9 @@ impl FileSystemRepository for FileSystem {
}
async fn cat(&self, request: CatRequest) -> Result<FileStream, CatError> {
self.cat(request.path())
.await
.map_err(|e| anyhow!("Failed to fetch file {}: {e:?}", request.path()).into())
self.cat(request.paths().paths()).await.map_err(|e| {
anyhow!("Failed to fetch files {:?}: {e:?}", request.paths().paths()).into()
})
}
async fn mkdir(&self, request: MkdirRequest) -> Result<(), MkdirError> {
@@ -462,16 +496,23 @@ where
let mut files = vec![];
while !dirs.is_empty() {
let mut dir_iter = tokio::fs::read_dir(dirs.remove(0)).await?;
let path = dirs.remove(0);
match tokio::fs::read_dir(path.clone()).await {
Ok(mut dir_iter) => {
while let Some(entry) = dir_iter.next_entry().await? {
let entry_path_buf = entry.path();
while let Some(entry) = dir_iter.next_entry().await? {
let entry_path_buf = entry.path();
if entry_path_buf.is_dir() {
dirs.push(entry_path_buf);
} else {
files.push(entry_path_buf);
if entry_path_buf.is_dir() {
dirs.push(entry_path_buf);
} else {
files.push(entry_path_buf);
}
}
}
Err(e) => match e.kind() {
std::io::ErrorKind::NotADirectory => files.push(path),
_ => return Err(e),
},
}
}
@@ -483,14 +524,12 @@ where
/// * `path`: The directory's path
/// * `tx`: The sender for the new ZIP archive's bytes
/// * `buffer_size`: The size of the file read buffer. A large buffer increases both the speed and the memory usage
async fn create_zip<P>(
path: P,
async fn create_zip(
prefix: String,
paths: Vec<PathBuf>,
tx: std::sync::mpsc::Sender<Result<bytes::Bytes, std::io::Error>>,
buffer_size: usize,
) -> anyhow::Result<()>
where
P: AsRef<Path>,
{
) -> anyhow::Result<()> {
let options = zip::write::SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Stored)
.compression_level(None)
@@ -500,12 +539,15 @@ where
let mut file_buf = vec![0; buffer_size];
let mut zip = zip::write::ZipWriter::new_stream(ChannelWriter(tx));
let entries = walk_dir(&path).await?;
let mut entries = Vec::new();
for path in &paths {
entries.append(&mut walk_dir(&path).await?);
}
for entry_path_buf in entries {
let entry_path = entry_path_buf.as_path();
let entry_str = entry_path
.strip_prefix(&path)?
.strip_prefix(&prefix)?
.to_str()
.context("Failed to get directory entry name")?;
@@ -546,3 +588,47 @@ impl Write for ChannelWriter {
Ok(())
}
}
enum PathRequest {
Single(FilePath),
Multiple {
lowest_common_prefix: String,
paths: Vec<FilePath>,
},
}
impl PathRequest {
fn from_paths(paths: Vec<FilePath>) -> anyhow::Result<Self> {
let mut input_paths: Vec<FilePath> = HashSet::<FilePath>::from_iter(paths.into_iter())
.into_iter()
.collect();
if input_paths.len() == 1 {
return Ok(Self::Single(input_paths.pop().unwrap()));
}
let mut lowest_common_prefix = input_paths
.first()
.expect("paths to contain at least 1 entry")
.to_string();
for path in &input_paths {
let chars = lowest_common_prefix
.chars()
.zip(path.as_str().chars())
.enumerate();
for (index, (a, b)) in chars {
if a != b {
lowest_common_prefix = lowest_common_prefix[..index].to_string();
break;
}
}
}
Ok(Self::Multiple {
lowest_common_prefix,
paths: input_paths,
})
}
}

21
backend/src/lib/outbound/notifier_debug_logger.rs
View File

@@ -8,7 +8,7 @@ use crate::domain::{
warren::{
models::{
auth_session::requests::FetchAuthSessionResponse,
file::{AbsoluteFilePath, LsResponse},
file::{AbsoluteFilePath, AbsoluteFilePathList, LsResponse},
share::{
CreateShareResponse, DeleteShareResponse, GetShareResponse, ListSharesResponse,
ShareCatResponse, ShareLsResponse,
@@ -58,10 +58,10 @@ impl WarrenNotifier for NotifierDebugLogger {
tracing::debug!("[Notifier] Fetched warren {}", warren.name());
}
async fn warren_cat(&self, warren: &Warren, path: &AbsoluteFilePath) {
async fn warren_cat(&self, warren: &Warren, paths: &AbsoluteFilePathList) {
tracing::debug!(
"[Notifier] Fetched file {} in warren {}",
path,
"[Notifier] Fetched {} file(s) in warren {}",
paths.paths().len(),
warren.name(),
);
}
@@ -165,8 +165,8 @@ impl WarrenNotifier for NotifierDebugLogger {
async fn warren_share_cat(&self, response: &ShareCatResponse) {
tracing::debug!(
"[Notifier] Fetched file {} from share {}",
response.path(),
"[Notifier] Fetched {} file(s) from share {}",
response.paths().paths().len(),
response.share().id(),
);
}
@@ -177,8 +177,8 @@ impl FileSystemNotifier for NotifierDebugLogger {
tracing::debug!("[Notifier] Listed {} file(s)", response.files().len());
}
async fn cat(&self, path: &AbsoluteFilePath) {
tracing::debug!("[Notifier] Fetched file {path}");
async fn cat(&self, paths: &AbsoluteFilePathList) {
tracing::debug!("[Notifier] Fetched {} file(s)", paths.paths().len());
}
async fn mkdir(&self, path: &AbsoluteFilePath) {
@@ -356,10 +356,11 @@ impl AuthNotifier for NotifierDebugLogger {
);
}
async fn auth_warren_cat(&self, user: &User, warren_id: &Uuid, path: &AbsoluteFilePath) {
async fn auth_warren_cat(&self, user: &User, warren_id: &Uuid, paths: &AbsoluteFilePathList) {
tracing::debug!(
"[Notifier] User {} fetched file {path} in warren {warren_id}",
"[Notifier] User {} fetched {} file(s) in warren {warren_id}",
user.id(),
paths.paths().len(),
);
}