Add support for uploading media files to S3

This commit is contained in:
Alex Auvolat 2023-05-12 13:19:41 +02:00
parent 1cb9459a23
commit 24c008b0de
4 changed files with 86 additions and 40 deletions

View file

@ -68,7 +68,7 @@ ructe = "0.15.0"
rsass = "0.26" rsass = "0.26"
[features] [features]
default = ["postgres"] default = ["postgres", "s3"]
postgres = ["plume-models/postgres", "diesel/postgres"] postgres = ["plume-models/postgres", "diesel/postgres"]
sqlite = ["plume-models/sqlite", "diesel/sqlite"] sqlite = ["plume-models/sqlite", "diesel/sqlite"]
debug-mailer = [] debug-mailer = []

View file

@ -171,11 +171,12 @@ impl Media {
pub fn delete(&self, conn: &Connection) -> Result<()> { pub fn delete(&self, conn: &Connection) -> Result<()> {
if !self.is_remote { if !self.is_remote {
if CONFIG.s3.is_some() { if CONFIG.s3.is_some() {
#[cfg(not(feature="s3"))]
unreachable!();
#[cfg(feature = "s3")] #[cfg(feature = "s3")]
CONFIG.s3.as_ref().unwrap().get_bucket() CONFIG.s3.as_ref().unwrap().get_bucket()
.delete_object_blocking(&self.file_path)?; .delete_object_blocking(&self.file_path)?;
#[cfg(not(feature="s3"))]
unreachable!();
} else { } else {
fs::remove_file(self.file_path.as_str())?; fs::remove_file(self.file_path.as_str())?;
} }

View file

@ -2,7 +2,7 @@ use crate::routes::{errors::ErrorPage, Page};
use crate::template_utils::{IntoContext, Ructe}; use crate::template_utils::{IntoContext, Ructe};
use guid_create::GUID; use guid_create::GUID;
use multipart::server::{ use multipart::server::{
save::{SaveResult, SavedData}, save::{SaveResult, SavedField, SavedData},
Multipart, Multipart,
}; };
use plume_models::{db_conn::DbConn, medias::*, users::User, Error, PlumeRocket, CONFIG}; use plume_models::{db_conn::DbConn, medias::*, users::User, Error, PlumeRocket, CONFIG};
@ -55,41 +55,16 @@ pub fn upload(
if let SaveResult::Full(entries) = Multipart::with_body(data.open(), boundary).save().temp() { if let SaveResult::Full(entries) = Multipart::with_body(data.open(), boundary).save().temp() {
let fields = entries.fields; let fields = entries.fields;
let filename = fields let file = fields
.get("file") .get("file")
.and_then(|v| v.iter().next()) .and_then(|v| v.iter().next())
.ok_or(status::BadRequest(Some("No file uploaded")))? .ok_or(status::BadRequest(Some("No file uploaded")))?;
.headers
.filename
.clone();
// Remove extension if it contains something else than just letters and numbers
let ext = filename
.and_then(|f| {
f.rsplit('.')
.next()
.and_then(|ext| {
if ext.chars().any(|c| !c.is_alphanumeric()) {
None
} else {
Some(ext.to_lowercase())
}
})
.map(|ext| format!(".{}", ext))
})
.unwrap_or_default();
let dest = format!("{}/{}{}", CONFIG.media_directory, GUID::rand(), ext);
match fields["file"][0].data { let file_path = match save_uploaded_file(file) {
SavedData::Bytes(ref bytes) => fs::write(&dest, bytes) Ok(Some(file_path)) => file_path,
.map_err(|_| status::BadRequest(Some("Couldn't save upload")))?, Ok(None) => return Ok(Redirect::to(uri!(new))),
SavedData::File(ref path, _) => { Err(_) => return Err(status::BadRequest(Some("Couldn't save uploaded media: {}"))),
fs::copy(path, &dest) };
.map_err(|_| status::BadRequest(Some("Couldn't copy upload")))?;
}
_ => {
return Ok(Redirect::to(uri!(new)));
}
}
let has_cw = !read(&fields["cw"][0].data) let has_cw = !read(&fields["cw"][0].data)
.map(|cw| cw.is_empty()) .map(|cw| cw.is_empty())
@ -97,7 +72,7 @@ pub fn upload(
let media = Media::insert( let media = Media::insert(
&conn, &conn,
NewMedia { NewMedia {
file_path: dest, file_path,
alt_text: read(&fields["alt"][0].data)?, alt_text: read(&fields["alt"][0].data)?,
is_remote: false, is_remote: false,
remote_url: None, remote_url: None,
@ -117,6 +92,75 @@ pub fn upload(
} }
} }
fn save_uploaded_file(file: &SavedField) -> Result<Option<String>, plume_models::Error> {
// Remove extension if it contains something else than just letters and numbers
let ext = file
.headers
.filename
.as_ref()
.and_then(|f| {
f.rsplit('.')
.next()
.and_then(|ext| {
if ext.chars().any(|c| !c.is_alphanumeric()) {
None
} else {
Some(ext.to_lowercase())
}
})
.map(|ext| format!(".{}", ext))
})
.unwrap_or_default();
if CONFIG.s3.is_some() {
#[cfg(not(feature="s3"))]
unreachable!();
#[cfg(feature="s3")]
{
use std::borrow::Cow;
let dest = format!("static/media/{}{}", GUID::rand(), ext);
let bytes = match file.data {
SavedData::Bytes(ref bytes) => Cow::from(bytes),
SavedData::File(ref path, _) => Cow::from(fs::read(path)?),
_ => {
return Ok(None);
}
};
let bucket = CONFIG.s3.as_ref().unwrap().get_bucket();
match &file.headers.content_type {
Some(ct) => {
bucket.put_object_with_content_type_blocking(&dest, &bytes, &ct.to_string())?;
}
None => {
bucket.put_object_blocking(&dest, &bytes)?;
}
}
Ok(Some(dest))
}
} else {
let dest = format!("{}/{}{}", CONFIG.media_directory, GUID::rand(), ext);
match file.data {
SavedData::Bytes(ref bytes) => {
fs::write(&dest, bytes)?;
}
SavedData::File(ref path, _) => {
fs::copy(path, &dest)?;
}
_ => {
return Ok(None);
}
}
Ok(Some(dest))
}
}
fn read(data: &SavedData) -> Result<String, status::BadRequest<&'static str>> { fn read(data: &SavedData) -> Result<String, status::BadRequest<&'static str>> {
if let SavedData::Text(s) = data { if let SavedData::Text(s) = data {
Ok(s.clone()) Ok(s.clone())

View file

@ -264,6 +264,9 @@ pub fn plume_static_files(file: PathBuf, build_id: &RawStr) -> Option<CachedFile
#[get("/static/media/<file..>")] #[get("/static/media/<file..>")]
pub fn plume_media_files(file: PathBuf) -> Option<CachedFile> { pub fn plume_media_files(file: PathBuf) -> Option<CachedFile> {
if CONFIG.s3.is_some() { if CONFIG.s3.is_some() {
#[cfg(not(feature="s3"))]
unreachable!();
#[cfg(feature="s3")] #[cfg(feature="s3")]
{ {
let ct = file.extension() let ct = file.extension()
@ -271,15 +274,13 @@ pub fn plume_media_files(file: PathBuf) -> Option<CachedFile> {
.unwrap_or(ContentType::Binary); .unwrap_or(ContentType::Binary);
let data = CONFIG.s3.as_ref().unwrap().get_bucket() let data = CONFIG.s3.as_ref().unwrap().get_bucket()
.get_object_blocking(format!("plume-media/{}", file.to_string_lossy())).ok()?; .get_object_blocking(format!("static/media/{}", file.to_string_lossy())).ok()?;
Some(CachedFile { Some(CachedFile {
inner: FileKind::S3 ( data.to_vec(), ct), inner: FileKind::S3 ( data.to_vec(), ct),
cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]),
}) })
} }
#[cfg(not(feature="s3"))]
unreachable!();
} else { } else {
NamedFile::open(Path::new(&CONFIG.media_directory).join(file)) NamedFile::open(Path::new(&CONFIG.media_directory).join(file))
.ok() .ok()