From 54af93d8fff19d55a55df8ad2ea26e9c31b27419 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sun, 13 Nov 2022 11:18:13 +0100 Subject: [PATCH 01/11] initial s3 support probably incomplete --- Cargo.lock | 244 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + plume-models/Cargo.toml | 1 + plume-models/src/config.rs | 108 +++++++++++++++- plume-models/src/lib.rs | 7 ++ plume-models/src/medias.rs | 7 +- src/routes/mod.rs | 37 ++++-- 7 files changed, 388 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a4e162d..a63c579f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,6 +115,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + [[package]] name = "ahash" version = "0.7.6" @@ -166,6 +172,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + [[package]] name = "arc-swap" version = "1.6.0" @@ -227,7 +239,7 @@ dependencies = [ "derive_builder", "diligent-date-parser", "never", - "quick-xml", + "quick-xml 0.27.1", ] [[package]] @@ -241,6 +253,22 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "attohttpc" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e69e13a99a7e6e070bb114f7ff381e58c7ccc188630121fc4c2fe4bcf24cd072" +dependencies = [ + "http 0.2.8", + "log 0.4.17", + "native-tls", + "openssl", + "serde 1.0.152", + "serde_json", + "url 2.3.1", + "wildmatch", +] + [[package]] name = "atty" version = "0.2.14" @@ -267,6 +295,31 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-creds" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460a75eac8f3cb7683e0a9a588a83c3ff039331ea7bfbfbfcecf1dacab276e11" +dependencies = [ + "anyhow", + "attohttpc", + "dirs", + "rust-ini 0.17.0", + "serde 1.0.152", + "serde-xml-rs", + "serde_derive", + "url 2.3.1", +] + +[[package]] +name = "aws-region" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10110ddbd800fb47e6bef95e88fc13495795d252f585272a4fa3ac4f5b2e0a4d" +dependencies = [ + "anyhow", +] + [[package]] name = "backtrace" version = "0.1.8" @@ -389,6 +442,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block_on_proc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b872f3528eeeb4370ee73b51194dc1cd93680c2d0eb6c7a223889038d2c1a167" +dependencies = [ + "quote 1.0.23", + "syn 1.0.107", +] + [[package]] name = "blowfish" version = "0.9.1" @@ -578,7 +641,7 @@ checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", "nom 5.1.2", - "rust-ini", + "rust-ini 0.13.0", "serde 1.0.152", "serde-hjson", "serde_json", @@ -636,7 +699,7 @@ dependencies = [ "aes-gcm", "base64 0.13.1", "hkdf", - "hmac", + "hmac 0.10.1", "percent-encoding 2.2.0", "rand 0.8.5", "sha2", @@ -876,6 +939,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "ctr" version = "0.6.0" @@ -1150,6 +1223,35 @@ dependencies = [ "chrono", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] +name = "dlv-list" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -1745,6 +1847,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash 0.4.7", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -1794,7 +1905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" dependencies = [ "digest", - "hmac", + "hmac 0.10.1", ] [[package]] @@ -1803,7 +1914,17 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ - "crypto-mac", + "crypto-mac 0.10.1", + "digest", +] + +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac 0.11.1", "digest", ] @@ -2549,6 +2670,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2 1.0.49", + "quote 1.0.23", + "syn 1.0.107", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -2641,6 +2773,15 @@ dependencies = [ "unicase 2.6.0", ] +[[package]] +name = "minidom" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" +dependencies = [ + "quick-xml 0.20.0", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3063,6 +3204,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" +dependencies = [ + "dlv-list", + "hashbrown 0.9.1", +] + [[package]] name = "overload" version = "0.1.1" @@ -3294,6 +3445,7 @@ dependencies = [ "rocket_i18n", "rsass", "ructe", + "rust-s3", "scheduled-thread-pool", "serde 1.0.152", "serde_json", @@ -3409,6 +3561,7 @@ dependencies = [ "riker", "rocket", "rocket_i18n", + "rust-s3", "scheduled-thread-pool", "serde 1.0.152", "serde_derive", @@ -3552,6 +3705,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.27.1" @@ -3836,6 +3998,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom 0.2.8", + "redox_syscall 0.2.16", + "thiserror", +] + [[package]] name = "regex" version = "1.7.0" @@ -3967,6 +4140,7 @@ dependencies = [ "tokio 1.24.1", "tokio-native-tls", "tokio-socks", + "tokio-util 0.7.4", "tower-service", "url 2.3.1", "wasm-bindgen", @@ -4159,6 +4333,48 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" +[[package]] +name = "rust-ini" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a4e82923ed07143871571852a390742200607e5058ce633afec89752f9c3f82" +dependencies = [ + "anyhow", + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.1", + "block_on_proc", + "cfg-if 1.0.0", + "hex", + "hmac 0.11.0", + "http 0.2.8", + "log 0.4.17", + "maybe-async", + "md5", + "minidom", + "percent-encoding 2.2.0", + "reqwest 0.11.13", + "serde 1.0.152", + "serde-xml-rs", + "serde_derive", + "sha2", + "time 0.3.17", + "tokio 1.24.1", + "tokio-stream", + "url 2.3.1", +] + [[package]] name = "rust-stemmers" version = "1.2.0" @@ -4302,6 +4518,18 @@ dependencies = [ "serde 0.8.23", ] +[[package]] +name = "serde-xml-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa" +dependencies = [ + "log 0.4.17", + "serde 1.0.152", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.152" @@ -5679,6 +5907,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "wildmatch" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee583bdc5ff1cf9db20e9db5bb3ff4c3089a8f6b8b31aff265c9aba85812db86" + [[package]] name = "winapi" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index 2c2e1d30..693c5d0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ rocket = "0.4.11" rocket_contrib = { version = "0.4.11", features = ["json"] } rocket_i18n = "0.4.1" scheduled-thread-pool = "0.2.6" +rust-s3 = { version = "0.29.0", no-default-features = true, features = ["blocking"], optional = true} serde = "1.0.137" serde_json = "1.0.81" shrinkwraprs = "0.3.0" @@ -74,6 +75,7 @@ sqlite = ["plume-models/sqlite", "diesel/sqlite"] debug-mailer = [] test = [] search-lindera = ["plume-models/search-lindera"] +s3 = ["rust-s3"] [workspace] members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"] diff --git a/plume-models/Cargo.toml b/plume-models/Cargo.toml index 914bf803..c28a6708 100644 --- a/plume-models/Cargo.toml +++ b/plume-models/Cargo.toml @@ -18,6 +18,7 @@ rocket_i18n = "0.4.1" reqwest = "0.11.11" scheduled-thread-pool = "0.2.6" serde = "1.0.137" +rust-s3 = { version = "0.29.0", no-default-features = true, features = ["blocking"] } serde_derive = "1.0" serde_json = "1.0.81" tantivy = "0.13.3" diff --git a/plume-models/src/config.rs b/plume-models/src/config.rs index ba705d42..d9ba0ea7 100644 --- a/plume-models/src/config.rs +++ b/plume-models/src/config.rs @@ -6,6 +6,10 @@ use rocket::Config as RocketConfig; use std::collections::HashSet; use std::env::{self, var}; +use s3::{Bucket, Region}; +use s3::creds::Credentials; + + #[cfg(not(test))] const DB_NAME: &str = "plume"; #[cfg(test)] @@ -27,13 +31,23 @@ pub struct Config { pub mail: Option, pub ldap: Option, pub proxy: Option, + pub s3: Option, } + impl Config { pub fn proxy(&self) -> Option<&reqwest::Proxy> { self.proxy.as_ref().map(|p| &p.proxy) } } +fn string_to_bool(val: &str, name: &str) -> bool { + match val { + "1" | "true" | "TRUE" => true, + "0" | "false" | "FALSE" => false, + _ => panic!("Invalid configuration: {} is not boolean", name), + } +} + #[derive(Debug, Clone)] pub enum InvalidRocketConfig { Env, @@ -288,11 +302,7 @@ fn get_ldap_config() -> Option { match (addr, base_dn) { (Some(addr), Some(base_dn)) => { let tls = var("LDAP_TLS").unwrap_or_else(|_| "false".to_owned()); - let tls = match tls.as_ref() { - "1" | "true" | "TRUE" => true, - "0" | "false" | "FALSE" => false, - _ => panic!("Invalid LDAP configuration : tls"), - }; + let tls = string_to_bool(&tls, "LDAP_TLS"); let user_name_attr = var("LDAP_USER_NAME_ATTR").unwrap_or_else(|_| "cn".to_owned()); let mail_attr = var("LDAP_USER_MAIL_ATTR").unwrap_or_else(|_| "mail".to_owned()); Some(LdapConfig { @@ -349,6 +359,93 @@ fn get_proxy_config() -> Option { }) } +pub struct S3Config { + pub bucket: String, + pub access_key_id: String, + pub access_key_secret: String, + + // region? If not set, default to us-east-1 + pub region: String, + // hostname for s3. If not set, default to $region.amazonaws.com + pub hostname: String, + // may be useful when using self hosted s3. Won't work with recent AWS buckets + pub path_style: bool, + pub protocol: String, + + // options below this comment are not used yet + // upload directly from user to S3, without going through Plume. Uses PostObject endpoint + pub direct_upload: bool, + // download directly from s3 to user, wihout going through Plume. Require public read on bucket + pub direct_download: bool, + // use this hostname for downloads, can be used with caching proxy in front of s3 + pub alias: Option, +} + +impl S3Config { + pub fn get_bucket(&self) -> Bucket { + let region = Region::Custom { + region: self.region.clone(), + endpoint: format!("{}://{}", self.protocol, self.hostname), + }; + let credentials = Credentials { + access_key: Some(self.access_key_id.clone()), + secret_key: Some(self.access_key_secret.clone()), + security_token: None, + session_token: None, + }; + + if self.path_style { + Bucket::new_with_path_style(&self.bucket, region, credentials) + } else { + Bucket::new(&self.bucket, region, credentials) + }.unwrap() + } +} + +fn get_s3_config() -> Option { + let bucket = var("S3_BUCKET").ok(); + let access_key_id = var("AWS_ACCESS_KEY_ID").ok(); + let access_key_secret = var("AWS_SECRET_ACCESS_KEY").ok(); + if bucket.is_none() && access_key_id.is_none() && access_key_secret.is_none() { + return None; + } + if bucket.is_none() || access_key_id.is_none() || access_key_secret.is_none() { + panic!("Invalid S3 configuration: some required values are set, but not others"); + } + let bucket = bucket.unwrap(); + let access_key_id = access_key_id.unwrap(); + let access_key_secret = access_key_secret.unwrap(); + + let region = var("S3_REGION").unwrap_or_else(|_| "us-east-1".to_owned()); + let hostname = var("S3_HOSTNAME").unwrap_or_else(|_| format!("{}.amazonaws.com", region)); + + let protocol = var("S3_PROTOCOL").unwrap_or_else(|_| "https".to_owned()); + if protocol != "http" && protocol != "https" { + panic!("Invalid S3 configuration: invalid protocol {}", protocol); + } + + let path_style = var("S3_PATH_STYLE").unwrap_or_else(|_| "false".to_owned()); + let path_style = string_to_bool(&path_style, "S3_PATH_STYLE"); + let direct_upload = var("S3_DIRECT_UPLOAD").unwrap_or_else(|_| "false".to_owned()); + let direct_upload = string_to_bool(&direct_upload, "S3_DIRECT_UPLOAD"); + let direct_download = var("S3_DIRECT_DOWNLOAD").unwrap_or_else(|_| "false".to_owned()); + let direct_download = string_to_bool(&direct_download, "S3_DIRECT_DOWNLOAD"); + + let alias = var("S3_ALIAS_HOST").ok(); + Some(S3Config { + bucket, + access_key_id, + access_key_secret, + region, + hostname, + protocol, + path_style, + direct_upload, + direct_download, + alias, + }) +} + lazy_static! { pub static ref CONFIG: Config = Config { base_url: var("BASE_URL").unwrap_or_else(|_| format!( @@ -380,5 +477,6 @@ lazy_static! { mail: get_mail_config(), ldap: get_ldap_config(), proxy: get_proxy_config(), + s3: get_s3_config(), }; } diff --git a/plume-models/src/lib.rs b/plume-models/src/lib.rs index e74a201c..b3fa19d9 100644 --- a/plume-models/src/lib.rs +++ b/plume-models/src/lib.rs @@ -69,6 +69,7 @@ pub enum Error { Webfinger, Expired, UserAlreadyExists, + Anyhow(anyhow::Error), } impl From for Error { @@ -170,6 +171,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: anyhow::Error) -> Error { + Error::Anyhow(err) + } +} + pub type Result = std::result::Result; /// Adds a function to a model, that returns the first diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index d3474a2b..192e60da 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -170,7 +170,12 @@ impl Media { pub fn delete(&self, conn: &Connection) -> Result<()> { if !self.is_remote { - fs::remove_file(self.file_path.as_str())?; + if let Some(config) = &CONFIG.s3 { + config.get_bucket() + .delete_object_blocking(&self.file_path)?; + } else { + fs::remove_file(self.file_path.as_str())?; + } } diesel::delete(self) .execute(conn) diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 58d71aa6..85fb4b89 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -9,7 +9,7 @@ use rocket::{ http::{ hyper::header::{CacheControl, CacheDirective, ETag, EntityTag}, uri::{FromUriParam, Query}, - RawStr, Status, + ContentType, RawStr, Status, }, request::{self, FromFormValue, FromRequest, Request}, response::{self, Flash, NamedFile, Redirect, Responder, Response}, @@ -204,10 +204,16 @@ pub mod timelines; pub mod user; pub mod well_known; +#[derive(Responder)] +enum FileKind { + Local(NamedFile), + S3(Vec, ContentType), +} + #[derive(Responder)] #[response()] pub struct CachedFile { - inner: NamedFile, + inner: FileKind, cache_control: CacheControl, } @@ -253,19 +259,36 @@ pub fn plume_static_files(file: PathBuf, build_id: &RawStr) -> Option")] pub fn plume_media_files(file: PathBuf) -> Option { - NamedFile::open(Path::new(&CONFIG.media_directory).join(file)) - .ok() - .map(|f| CachedFile { - inner: f, + if let Some(config) = &CONFIG.s3 { + let ct = file.extension() + .and_then(|ext| ContentType::from_extension(&ext.to_string_lossy())) + .unwrap_or(ContentType::Binary); + + let (data, code) = config.get_bucket() + .get_object_blocking(format!("plume-media/{}", file.to_string_lossy())).ok()?; + if code != 200 { + return None; + } + + Some(CachedFile { + inner: FileKind::S3 ( data, ct), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), }) + } else { + NamedFile::open(Path::new(&CONFIG.media_directory).join(file)) + .ok() + .map(|f| CachedFile { + inner: FileKind::Local(f), + cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), + }) + } } #[get("/static/", rank = 3)] pub fn static_files(file: PathBuf) -> Option { NamedFile::open(Path::new("static/").join(file)) .ok() .map(|f| CachedFile { - inner: f, + inner: FileKind::Local(f), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), }) } From 30a3cec87e6d5f90dc033955231b24d197eea03e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 11:41:38 +0200 Subject: [PATCH 02/11] Add Nix development shell --- .envrc | 1 + .gitignore | 1 + flake.lock | 130 +++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 24 ++++++++++ 4 files changed, 156 insertions(+) create mode 100644 .envrc create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..3550a30f --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore index bd576f31..fed4946b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ search_index __pycache__ .vscode/ *-journal +.direnv/ diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..b8334cbb --- /dev/null +++ b/flake.lock @@ -0,0 +1,130 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1683408522, + "narHash": "sha256-9kcPh6Uxo17a3kK3XCHhcWiV1Yu1kYj22RHiymUhMkU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "897876e4c484f1e8f92009fd11b7d988a121a4e7", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1681358109, + "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1683857898, + "narHash": "sha256-pyVY4UxM6zUX97g6bk6UyCbZGCWZb2Zykrne8YxacRA=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "4e7fba3f37f5e184ada0ef3cf1e4d8ef450f240b", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..2646d14c --- /dev/null +++ b/flake.nix @@ -0,0 +1,24 @@ +{ + description = "Developpment shell for Plume including nightly Rust compiler"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.rust-overlay.url = "github:oxalica/rust-overlay"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils, rust-overlay, ... }: + flake-utils.lib.eachDefaultSystem (system: + let + overlays = [ (import rust-overlay) ]; + pkgs = import nixpkgs { inherit system overlays; }; + in { + devShells.default = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + rust-bin.nightly.latest.default + openssl + pkg-config + gettext + postgresql + ]; + }; + }); +} From 10e06737cf576799c3906a6c8f38bf430321d4fa Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 12:12:32 +0200 Subject: [PATCH 03/11] Update rust-s3 dependency and move Cargo.toml dependencies --- Cargo.lock | 195 +++++++++++++++++++------------------ Cargo.toml | 6 +- plume-models/Cargo.toml | 4 +- plume-models/src/config.rs | 8 +- plume-models/src/lib.rs | 10 +- src/routes/mod.rs | 7 +- 6 files changed, 120 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a63c579f..27a833d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,12 +115,6 @@ dependencies = [ "const-random", ] -[[package]] -name = "ahash" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" - [[package]] name = "ahash" version = "0.7.6" @@ -172,12 +166,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "anyhow" -version = "1.0.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" - [[package]] name = "arc-swap" version = "1.6.0" @@ -255,18 +243,16 @@ dependencies = [ [[package]] name = "attohttpc" -version = "0.18.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e69e13a99a7e6e070bb114f7ff381e58c7ccc188630121fc4c2fe4bcf24cd072" +checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" dependencies = [ "http 0.2.8", "log 0.4.17", "native-tls", - "openssl", "serde 1.0.152", "serde_json", "url 2.3.1", - "wildmatch", ] [[package]] @@ -297,27 +283,28 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-creds" -version = "0.27.1" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460a75eac8f3cb7683e0a9a588a83c3ff039331ea7bfbfbfcecf1dacab276e11" +checksum = "3776743bb68d4ad02ba30ba8f64373f1be4e082fe47651767171ce75bb2f6cf5" dependencies = [ - "anyhow", "attohttpc", "dirs", - "rust-ini 0.17.0", + "log 0.4.17", + "quick-xml 0.26.0", + "rust-ini 0.18.0", "serde 1.0.152", - "serde-xml-rs", - "serde_derive", + "thiserror", + "time 0.3.17", "url 2.3.1", ] [[package]] name = "aws-region" -version = "0.23.5" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10110ddbd800fb47e6bef95e88fc13495795d252f585272a4fa3ac4f5b2e0a4d" +checksum = "056557a61427d0e5ba29dd931031c8ffed4ee7a550e7cd55692a9d8deb0a9dba" dependencies = [ - "anyhow", + "thiserror", ] [[package]] @@ -442,6 +429,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block_on_proc" version = "0.2.0" @@ -702,7 +698,7 @@ dependencies = [ "hmac 0.10.1", "percent-encoding 2.2.0", "rand 0.8.5", - "sha2", + "sha2 0.9.9", "time 0.1.45", ] @@ -939,16 +935,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "crypto-mac" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "ctr" version = "0.6.0" @@ -1214,6 +1200,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common", + "subtle", +] + [[package]] name = "diligent-date-parser" version = "0.1.4" @@ -1245,12 +1242,9 @@ dependencies = [ [[package]] name = "dlv-list" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b" -dependencies = [ - "rand 0.8.5", -] +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" [[package]] name = "dotenv" @@ -1847,15 +1841,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" -dependencies = [ - "ahash 0.4.7", -] - [[package]] name = "hashbrown" version = "0.11.2" @@ -1904,7 +1889,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" dependencies = [ - "digest", + "digest 0.9.0", "hmac 0.10.1", ] @@ -1914,18 +1899,17 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ - "crypto-mac 0.10.1", - "digest", + "crypto-mac", + "digest 0.9.0", ] [[package]] name = "hmac" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "crypto-mac 0.11.1", - "digest", + "digest 0.10.6", ] [[package]] @@ -2775,11 +2759,11 @@ dependencies = [ [[package]] name = "minidom" -version = "0.13.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" +checksum = "2e9ce45d459e358790a285e7609ff5ae4cfab88b75f237e8838e62029dda397b" dependencies = [ - "quick-xml 0.20.0", + "rxml", ] [[package]] @@ -3206,12 +3190,12 @@ dependencies = [ [[package]] name = "ordered-multimap" -version = "0.3.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" dependencies = [ "dlv-list", - "hashbrown 0.9.1", + "hashbrown 0.12.3", ] [[package]] @@ -3445,7 +3429,6 @@ dependencies = [ "rocket_i18n", "rsass", "ructe", - "rust-s3", "scheduled-thread-pool", "serde 1.0.152", "serde_json", @@ -3707,11 +3690,12 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.20.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" dependencies = [ "memchr", + "serde 1.0.152", ] [[package]] @@ -4335,9 +4319,9 @@ checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" [[package]] name = "rust-ini" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" dependencies = [ "cfg-if 1.0.0", "ordered-multimap", @@ -4345,30 +4329,32 @@ dependencies = [ [[package]] name = "rust-s3" -version = "0.29.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4e82923ed07143871571852a390742200607e5058ce633afec89752f9c3f82" +checksum = "1b2ac5ff6acfbe74226fa701b5ef793aaa054055c13ebb7060ad36942956e027" dependencies = [ - "anyhow", "async-trait", "aws-creds", "aws-region", "base64 0.13.1", "block_on_proc", + "bytes 1.3.0", "cfg-if 1.0.0", + "futures 0.3.25", "hex", - "hmac 0.11.0", + "hmac 0.12.1", "http 0.2.8", "log 0.4.17", "maybe-async", "md5", "minidom", "percent-encoding 2.2.0", + "quick-xml 0.26.0", "reqwest 0.11.13", "serde 1.0.152", - "serde-xml-rs", "serde_derive", - "sha2", + "sha2 0.10.6", + "thiserror", "time 0.3.17", "tokio 1.24.1", "tokio-stream", @@ -4400,6 +4386,25 @@ dependencies = [ "semver", ] +[[package]] +name = "rxml" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a071866b8c681dc2cfffa77184adc32b57b0caad4e620b6292609703bceb804" +dependencies = [ + "bytes 1.3.0", + "pin-project-lite 0.2.9", + "rxml_validation", + "smartstring", + "tokio 1.24.1", +] + +[[package]] +name = "rxml_validation" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53bc79743f9a66c2fb1f951cd83735f275d46bfe466259fbc5897bb60a0d00ee" + [[package]] name = "ryu" version = "1.0.12" @@ -4518,18 +4523,6 @@ dependencies = [ "serde 0.8.23", ] -[[package]] -name = "serde-xml-rs" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa" -dependencies = [ - "log 0.4.17", - "serde 1.0.152", - "thiserror", - "xml-rs", -] - [[package]] name = "serde_derive" version = "1.0.152" @@ -4591,13 +4584,24 @@ version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.9.0", "opaque-debug", ] +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.6", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -4687,6 +4691,15 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "smartstring" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e714dff2b33f2321fdcd475b71cec79781a692d846f37f415fb395a1d2bcd48e" +dependencies = [ + "static_assertions", +] + [[package]] name = "snap" version = "1.1.0" @@ -5907,12 +5920,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "wildmatch" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee583bdc5ff1cf9db20e9db5bb3ff4c3089a8f6b8b31aff265c9aba85812db86" - [[package]] name = "winapi" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index 693c5d0e..b88325b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ rocket = "0.4.11" rocket_contrib = { version = "0.4.11", features = ["json"] } rocket_i18n = "0.4.1" scheduled-thread-pool = "0.2.6" -rust-s3 = { version = "0.29.0", no-default-features = true, features = ["blocking"], optional = true} +#aws-creds = { version = "0.34", default-features = false, features = ["native-tls"] } serde = "1.0.137" serde_json = "1.0.81" shrinkwraprs = "0.3.0" @@ -69,13 +69,13 @@ ructe = "0.15.0" rsass = "0.26" [features] -default = ["postgres"] +default = ["postgres", "s3"] postgres = ["plume-models/postgres", "diesel/postgres"] sqlite = ["plume-models/sqlite", "diesel/sqlite"] debug-mailer = [] test = [] search-lindera = ["plume-models/search-lindera"] -s3 = ["rust-s3"] +s3 = ["plume-models/s3"] [workspace] members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"] diff --git a/plume-models/Cargo.toml b/plume-models/Cargo.toml index c28a6708..5e8dfd83 100644 --- a/plume-models/Cargo.toml +++ b/plume-models/Cargo.toml @@ -18,7 +18,8 @@ rocket_i18n = "0.4.1" reqwest = "0.11.11" scheduled-thread-pool = "0.2.6" serde = "1.0.137" -rust-s3 = { version = "0.29.0", no-default-features = true, features = ["blocking"] } +#rust-s3 = { version = "0.29.0", default-features = false, features = ["blocking"] } +rust-s3 = { version = "0.33.0", optional = true, features = ["blocking"] } serde_derive = "1.0" serde_json = "1.0.81" tantivy = "0.13.3" @@ -62,3 +63,4 @@ diesel_migrations = "1.3.0" postgres = ["diesel/postgres", "plume-macro/postgres" ] sqlite = ["diesel/sqlite", "plume-macro/sqlite" ] search-lindera = ["lindera-tantivy"] +s3 = ["rust-s3"] diff --git a/plume-models/src/config.rs b/plume-models/src/config.rs index d9ba0ea7..af2605ae 100644 --- a/plume-models/src/config.rs +++ b/plume-models/src/config.rs @@ -392,13 +392,15 @@ impl S3Config { secret_key: Some(self.access_key_secret.clone()), security_token: None, session_token: None, + expiration: None, }; + let bucket = Bucket::new(&self.bucket, region, credentials).unwrap(); if self.path_style { - Bucket::new_with_path_style(&self.bucket, region, credentials) + bucket.with_path_style() } else { - Bucket::new(&self.bucket, region, credentials) - }.unwrap() + bucket + } } } diff --git a/plume-models/src/lib.rs b/plume-models/src/lib.rs index b3fa19d9..7afbeabb 100644 --- a/plume-models/src/lib.rs +++ b/plume-models/src/lib.rs @@ -69,7 +69,8 @@ pub enum Error { Webfinger, Expired, UserAlreadyExists, - Anyhow(anyhow::Error), + #[cfg(feature = "s3")] + S3(s3::error::S3Error), } impl From for Error { @@ -171,9 +172,10 @@ impl From for Error { } } -impl From for Error { - fn from(err: anyhow::Error) -> Error { - Error::Anyhow(err) +#[cfg(feature = "s3")] +impl From for Error { + fn from(err: s3::error::S3Error) -> Error { + Error::S3(err) } } diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 85fb4b89..7f49b399 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -264,14 +264,11 @@ pub fn plume_media_files(file: PathBuf) -> Option { .and_then(|ext| ContentType::from_extension(&ext.to_string_lossy())) .unwrap_or(ContentType::Binary); - let (data, code) = config.get_bucket() + let data = config.get_bucket() .get_object_blocking(format!("plume-media/{}", file.to_string_lossy())).ok()?; - if code != 200 { - return None; - } Some(CachedFile { - inner: FileKind::S3 ( data, ct), + inner: FileKind::S3 ( data.to_vec(), ct), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), }) } else { From 1cb9459a23f34a4a5cb82c7ef5c7c60a73d40a56 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 12:28:00 +0200 Subject: [PATCH 04/11] Update S3 features and make S3 support optional --- Cargo.toml | 3 +- plume-models/Cargo.toml | 1 - plume-models/src/config.rs | 82 +++++++++++++++++++++----------------- plume-models/src/medias.rs | 7 +++- src/routes/mod.rs | 31 +++++++++----- 5 files changed, 71 insertions(+), 53 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b88325b3..d2fd4775 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ rocket = "0.4.11" rocket_contrib = { version = "0.4.11", features = ["json"] } rocket_i18n = "0.4.1" scheduled-thread-pool = "0.2.6" -#aws-creds = { version = "0.34", default-features = false, features = ["native-tls"] } serde = "1.0.137" serde_json = "1.0.81" shrinkwraprs = "0.3.0" @@ -69,7 +68,7 @@ ructe = "0.15.0" rsass = "0.26" [features] -default = ["postgres", "s3"] +default = ["postgres"] postgres = ["plume-models/postgres", "diesel/postgres"] sqlite = ["plume-models/sqlite", "diesel/sqlite"] debug-mailer = [] diff --git a/plume-models/Cargo.toml b/plume-models/Cargo.toml index 5e8dfd83..6eac4b62 100644 --- a/plume-models/Cargo.toml +++ b/plume-models/Cargo.toml @@ -18,7 +18,6 @@ rocket_i18n = "0.4.1" reqwest = "0.11.11" scheduled-thread-pool = "0.2.6" serde = "1.0.137" -#rust-s3 = { version = "0.29.0", default-features = false, features = ["blocking"] } rust-s3 = { version = "0.33.0", optional = true, features = ["blocking"] } serde_derive = "1.0" serde_json = "1.0.81" diff --git a/plume-models/src/config.rs b/plume-models/src/config.rs index af2605ae..b6676f27 100644 --- a/plume-models/src/config.rs +++ b/plume-models/src/config.rs @@ -6,9 +6,8 @@ use rocket::Config as RocketConfig; use std::collections::HashSet; use std::env::{self, var}; -use s3::{Bucket, Region}; -use s3::creds::Credentials; - +#[cfg(feature = "s3")] +use s3::{Bucket, Region, creds::Credentials}; #[cfg(not(test))] const DB_NAME: &str = "plume"; @@ -382,6 +381,7 @@ pub struct S3Config { } impl S3Config { + #[cfg(feature = "s3")] pub fn get_bucket(&self) -> Bucket { let region = Region::Custom { region: self.region.clone(), @@ -411,41 +411,49 @@ fn get_s3_config() -> Option { if bucket.is_none() && access_key_id.is_none() && access_key_secret.is_none() { return None; } - if bucket.is_none() || access_key_id.is_none() || access_key_secret.is_none() { - panic!("Invalid S3 configuration: some required values are set, but not others"); + + #[cfg(not(feature = "s3"))] + panic!("S3 support is not enabled in this build"); + + #[cfg(feature = "s3")] + { + if bucket.is_none() || access_key_id.is_none() || access_key_secret.is_none() { + panic!("Invalid S3 configuration: some required values are set, but not others"); + } + let bucket = bucket.unwrap(); + let access_key_id = access_key_id.unwrap(); + let access_key_secret = access_key_secret.unwrap(); + + let region = var("S3_REGION").unwrap_or_else(|_| "us-east-1".to_owned()); + let hostname = var("S3_HOSTNAME").unwrap_or_else(|_| format!("{}.amazonaws.com", region)); + + let protocol = var("S3_PROTOCOL").unwrap_or_else(|_| "https".to_owned()); + if protocol != "http" && protocol != "https" { + panic!("Invalid S3 configuration: invalid protocol {}", protocol); + } + + let path_style = var("S3_PATH_STYLE").unwrap_or_else(|_| "false".to_owned()); + let path_style = string_to_bool(&path_style, "S3_PATH_STYLE"); + let direct_upload = var("S3_DIRECT_UPLOAD").unwrap_or_else(|_| "false".to_owned()); + let direct_upload = string_to_bool(&direct_upload, "S3_DIRECT_UPLOAD"); + let direct_download = var("S3_DIRECT_DOWNLOAD").unwrap_or_else(|_| "false".to_owned()); + let direct_download = string_to_bool(&direct_download, "S3_DIRECT_DOWNLOAD"); + + let alias = var("S3_ALIAS_HOST").ok(); + + Some(S3Config { + bucket, + access_key_id, + access_key_secret, + region, + hostname, + protocol, + path_style, + direct_upload, + direct_download, + alias, + }) } - let bucket = bucket.unwrap(); - let access_key_id = access_key_id.unwrap(); - let access_key_secret = access_key_secret.unwrap(); - - let region = var("S3_REGION").unwrap_or_else(|_| "us-east-1".to_owned()); - let hostname = var("S3_HOSTNAME").unwrap_or_else(|_| format!("{}.amazonaws.com", region)); - - let protocol = var("S3_PROTOCOL").unwrap_or_else(|_| "https".to_owned()); - if protocol != "http" && protocol != "https" { - panic!("Invalid S3 configuration: invalid protocol {}", protocol); - } - - let path_style = var("S3_PATH_STYLE").unwrap_or_else(|_| "false".to_owned()); - let path_style = string_to_bool(&path_style, "S3_PATH_STYLE"); - let direct_upload = var("S3_DIRECT_UPLOAD").unwrap_or_else(|_| "false".to_owned()); - let direct_upload = string_to_bool(&direct_upload, "S3_DIRECT_UPLOAD"); - let direct_download = var("S3_DIRECT_DOWNLOAD").unwrap_or_else(|_| "false".to_owned()); - let direct_download = string_to_bool(&direct_download, "S3_DIRECT_DOWNLOAD"); - - let alias = var("S3_ALIAS_HOST").ok(); - Some(S3Config { - bucket, - access_key_id, - access_key_secret, - region, - hostname, - protocol, - path_style, - direct_upload, - direct_download, - alias, - }) } lazy_static! { diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index 192e60da..3dc22605 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -170,9 +170,12 @@ impl Media { pub fn delete(&self, conn: &Connection) -> Result<()> { if !self.is_remote { - if let Some(config) = &CONFIG.s3 { - config.get_bucket() + if CONFIG.s3.is_some() { + #[cfg(feature = "s3")] + CONFIG.s3.as_ref().unwrap().get_bucket() .delete_object_blocking(&self.file_path)?; + #[cfg(not(feature="s3"))] + unreachable!(); } else { fs::remove_file(self.file_path.as_str())?; } diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 7f49b399..08fd8fa1 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -9,7 +9,7 @@ use rocket::{ http::{ hyper::header::{CacheControl, CacheDirective, ETag, EntityTag}, uri::{FromUriParam, Query}, - ContentType, RawStr, Status, + RawStr, Status, }, request::{self, FromFormValue, FromRequest, Request}, response::{self, Flash, NamedFile, Redirect, Responder, Response}, @@ -21,6 +21,9 @@ use std::{ path::{Path, PathBuf}, }; +#[cfg(feature = "s3")] +use rocket::http::ContentType; + /// Special return type used for routes that "cannot fail", and instead /// `Redirect`, or `Flash`, when we cannot deliver a `Ructe` Response #[allow(clippy::large_enum_variant)] @@ -207,6 +210,7 @@ pub mod well_known; #[derive(Responder)] enum FileKind { Local(NamedFile), + #[cfg(feature = "s3")] S3(Vec, ContentType), } @@ -259,18 +263,23 @@ pub fn plume_static_files(file: PathBuf, build_id: &RawStr) -> Option")] pub fn plume_media_files(file: PathBuf) -> Option { - if let Some(config) = &CONFIG.s3 { - let ct = file.extension() - .and_then(|ext| ContentType::from_extension(&ext.to_string_lossy())) - .unwrap_or(ContentType::Binary); + if CONFIG.s3.is_some() { + #[cfg(feature="s3")] + { + let ct = file.extension() + .and_then(|ext| ContentType::from_extension(&ext.to_string_lossy())) + .unwrap_or(ContentType::Binary); - let data = config.get_bucket() - .get_object_blocking(format!("plume-media/{}", file.to_string_lossy())).ok()?; + let data = CONFIG.s3.as_ref().unwrap().get_bucket() + .get_object_blocking(format!("plume-media/{}", file.to_string_lossy())).ok()?; - Some(CachedFile { - inner: FileKind::S3 ( data.to_vec(), ct), - cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), - }) + Some(CachedFile { + inner: FileKind::S3 ( data.to_vec(), ct), + cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), + }) + } + #[cfg(not(feature="s3"))] + unreachable!(); } else { NamedFile::open(Path::new(&CONFIG.media_directory).join(file)) .ok() From 24c008b0de3b9e1207874cc78a97dc97ba431dce Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 13:19:41 +0200 Subject: [PATCH 05/11] Add support for uploading media files to S3 --- Cargo.toml | 2 +- plume-models/src/medias.rs | 5 +- src/routes/medias.rs | 112 ++++++++++++++++++++++++++----------- src/routes/mod.rs | 7 ++- 4 files changed, 86 insertions(+), 40 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d2fd4775..f5868e32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ ructe = "0.15.0" rsass = "0.26" [features] -default = ["postgres"] +default = ["postgres", "s3"] postgres = ["plume-models/postgres", "diesel/postgres"] sqlite = ["plume-models/sqlite", "diesel/sqlite"] debug-mailer = [] diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index 3dc22605..60f0d2aa 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -171,11 +171,12 @@ impl Media { pub fn delete(&self, conn: &Connection) -> Result<()> { if !self.is_remote { if CONFIG.s3.is_some() { + #[cfg(not(feature="s3"))] + unreachable!(); + #[cfg(feature = "s3")] CONFIG.s3.as_ref().unwrap().get_bucket() .delete_object_blocking(&self.file_path)?; - #[cfg(not(feature="s3"))] - unreachable!(); } else { fs::remove_file(self.file_path.as_str())?; } diff --git a/src/routes/medias.rs b/src/routes/medias.rs index 1981c542..b9a22dee 100644 --- a/src/routes/medias.rs +++ b/src/routes/medias.rs @@ -2,7 +2,7 @@ use crate::routes::{errors::ErrorPage, Page}; use crate::template_utils::{IntoContext, Ructe}; use guid_create::GUID; use multipart::server::{ - save::{SaveResult, SavedData}, + save::{SaveResult, SavedField, SavedData}, Multipart, }; use plume_models::{db_conn::DbConn, medias::*, users::User, Error, PlumeRocket, CONFIG}; @@ -55,41 +55,16 @@ pub fn upload( if let SaveResult::Full(entries) = Multipart::with_body(data.open(), boundary).save().temp() { let fields = entries.fields; - let filename = fields + let file = fields .get("file") .and_then(|v| v.iter().next()) - .ok_or(status::BadRequest(Some("No file uploaded")))? - .headers - .filename - .clone(); - // Remove extension if it contains something else than just letters and numbers - let ext = filename - .and_then(|f| { - f.rsplit('.') - .next() - .and_then(|ext| { - if ext.chars().any(|c| !c.is_alphanumeric()) { - None - } else { - Some(ext.to_lowercase()) - } - }) - .map(|ext| format!(".{}", ext)) - }) - .unwrap_or_default(); - let dest = format!("{}/{}{}", CONFIG.media_directory, GUID::rand(), ext); + .ok_or(status::BadRequest(Some("No file uploaded")))?; - match fields["file"][0].data { - SavedData::Bytes(ref bytes) => fs::write(&dest, bytes) - .map_err(|_| status::BadRequest(Some("Couldn't save upload")))?, - SavedData::File(ref path, _) => { - fs::copy(path, &dest) - .map_err(|_| status::BadRequest(Some("Couldn't copy upload")))?; - } - _ => { - return Ok(Redirect::to(uri!(new))); - } - } + let file_path = match save_uploaded_file(file) { + Ok(Some(file_path)) => file_path, + Ok(None) => return Ok(Redirect::to(uri!(new))), + Err(_) => return Err(status::BadRequest(Some("Couldn't save uploaded media: {}"))), + }; let has_cw = !read(&fields["cw"][0].data) .map(|cw| cw.is_empty()) @@ -97,7 +72,7 @@ pub fn upload( let media = Media::insert( &conn, NewMedia { - file_path: dest, + file_path, alt_text: read(&fields["alt"][0].data)?, is_remote: false, remote_url: None, @@ -117,6 +92,75 @@ pub fn upload( } } +fn save_uploaded_file(file: &SavedField) -> Result, plume_models::Error> { + // Remove extension if it contains something else than just letters and numbers + let ext = file + .headers + .filename + .as_ref() + .and_then(|f| { + f.rsplit('.') + .next() + .and_then(|ext| { + if ext.chars().any(|c| !c.is_alphanumeric()) { + None + } else { + Some(ext.to_lowercase()) + } + }) + .map(|ext| format!(".{}", ext)) + }) + .unwrap_or_default(); + + if CONFIG.s3.is_some() { + #[cfg(not(feature="s3"))] + unreachable!(); + + #[cfg(feature="s3")] + { + use std::borrow::Cow; + + let dest = format!("static/media/{}{}", GUID::rand(), ext); + + let bytes = match file.data { + SavedData::Bytes(ref bytes) => Cow::from(bytes), + SavedData::File(ref path, _) => Cow::from(fs::read(path)?), + _ => { + return Ok(None); + } + }; + + let bucket = CONFIG.s3.as_ref().unwrap().get_bucket(); + match &file.headers.content_type { + Some(ct) => { + bucket.put_object_with_content_type_blocking(&dest, &bytes, &ct.to_string())?; + } + None => { + bucket.put_object_blocking(&dest, &bytes)?; + } + } + + Ok(Some(dest)) + } + } else { + let dest = format!("{}/{}{}", CONFIG.media_directory, GUID::rand(), ext); + + match file.data { + SavedData::Bytes(ref bytes) => { + fs::write(&dest, bytes)?; + } + SavedData::File(ref path, _) => { + fs::copy(path, &dest)?; + } + _ => { + return Ok(None); + } + } + + Ok(Some(dest)) + } +} + fn read(data: &SavedData) -> Result> { if let SavedData::Text(s) = data { Ok(s.clone()) diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 08fd8fa1..ac016b9d 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -264,6 +264,9 @@ pub fn plume_static_files(file: PathBuf, build_id: &RawStr) -> Option")] pub fn plume_media_files(file: PathBuf) -> Option { if CONFIG.s3.is_some() { + #[cfg(not(feature="s3"))] + unreachable!(); + #[cfg(feature="s3")] { let ct = file.extension() @@ -271,15 +274,13 @@ pub fn plume_media_files(file: PathBuf) -> Option { .unwrap_or(ContentType::Binary); let data = CONFIG.s3.as_ref().unwrap().get_bucket() - .get_object_blocking(format!("plume-media/{}", file.to_string_lossy())).ok()?; + .get_object_blocking(format!("static/media/{}", file.to_string_lossy())).ok()?; Some(CachedFile { inner: FileKind::S3 ( data.to_vec(), ct), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), }) } - #[cfg(not(feature="s3"))] - unreachable!(); } else { NamedFile::open(Path::new(&CONFIG.media_directory).join(file)) .ok() From 4e67eb83170983ed19f837350311f5dc549830df Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 15:40:36 +0200 Subject: [PATCH 06/11] Uniformize media path/URL handling and implement direct download from S3 backend --- plume-models/src/config.rs | 13 +++-- plume-models/src/medias.rs | 107 ++++++++++++++++++++++++++++++------- 2 files changed, 94 insertions(+), 26 deletions(-) diff --git a/plume-models/src/config.rs b/plume-models/src/config.rs index b6676f27..dc27497b 100644 --- a/plume-models/src/config.rs +++ b/plume-models/src/config.rs @@ -371,12 +371,10 @@ pub struct S3Config { pub path_style: bool, pub protocol: String, - // options below this comment are not used yet - // upload directly from user to S3, without going through Plume. Uses PostObject endpoint - pub direct_upload: bool, // download directly from s3 to user, wihout going through Plume. Require public read on bucket pub direct_download: bool, - // use this hostname for downloads, can be used with caching proxy in front of s3 + // use this hostname for downloads, can be used with caching proxy in front of s3 (expected to + // be reachable through https) pub alias: Option, } @@ -434,13 +432,15 @@ fn get_s3_config() -> Option { let path_style = var("S3_PATH_STYLE").unwrap_or_else(|_| "false".to_owned()); let path_style = string_to_bool(&path_style, "S3_PATH_STYLE"); - let direct_upload = var("S3_DIRECT_UPLOAD").unwrap_or_else(|_| "false".to_owned()); - let direct_upload = string_to_bool(&direct_upload, "S3_DIRECT_UPLOAD"); let direct_download = var("S3_DIRECT_DOWNLOAD").unwrap_or_else(|_| "false".to_owned()); let direct_download = string_to_bool(&direct_download, "S3_DIRECT_DOWNLOAD"); let alias = var("S3_ALIAS_HOST").ok(); + if direct_download && protocol == "http" && alias.is_none() { + panic!("S3 direct download is disabled because bucket is accessed through plain HTTP. Use HTTPS or set an alias hostname (S3_ALIAS_HOST)."); + } + Some(S3Config { bucket, access_key_id, @@ -449,7 +449,6 @@ fn get_s3_config() -> Option { hostname, protocol, path_style, - direct_upload, direct_download, alias, }) diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index 60f0d2aa..f371cbab 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -16,6 +16,9 @@ use std::{ use tracing::warn; use url::Url; +#[cfg(feature = "s3")] +use crate::config::S3Config; + const REMOTE_MEDIA_DIRECTORY: &str = "remote"; #[derive(Clone, Identifiable, Queryable, AsChangeset)] @@ -105,7 +108,7 @@ impl Media { .file_path .rsplit_once('.') .map(|x| x.1) - .expect("Media::category: extension error") + .unwrap_or("") .to_lowercase() { "png" | "jpg" | "jpeg" | "gif" | "svg" => MediaCategory::Image, @@ -151,19 +154,83 @@ impl Media { }) } + /// Returns full file path for medias stored in the local media directory. + pub fn local_path(&self) -> Option { + if self.file_path.is_empty() { + return None; + } + + if CONFIG.s3.is_some() { + #[cfg(feature="s3")] + unreachable!("Called Media::local_path() but media are stored on S3"); + #[cfg(not(feature="s3"))] + unreachable!(); + } + + let relative_path = self + .file_path + .trim_start_matches(&CONFIG.media_directory) + .trim_start_matches(path::MAIN_SEPARATOR) + .trim_start_matches("static/media/"); + + Some(Path::new(&CONFIG.media_directory).join(relative_path)) + } + + /// Returns the relative URL to access this file, which is also the key at which + /// it is stored in the S3 bucket if we are using S3 storage. + /// Does not start with a '/', it is of the form "static/media/<...>" + pub fn relative_url(&self) -> Option { + if self.file_path.is_empty() { + return None; + } + + let relative_path = self + .file_path + .trim_start_matches(&CONFIG.media_directory) + .replace(path::MAIN_SEPARATOR, "/"); + + let relative_path = relative_path + .trim_start_matches('/') + .trim_start_matches("static/media/"); + + Some(format!("static/media/{}", relative_path)) + } + + /// Returns a public URL through which this media file can be accessed pub fn url(&self) -> Result { if self.is_remote { Ok(self.remote_url.clone().unwrap_or_default()) } else { - let file_path = self.file_path.replace(path::MAIN_SEPARATOR, "/").replacen( - &CONFIG.media_directory, - "static/media", - 1, - ); // "static/media" from plume::routs::plume_media_files() + let relative_url = self.relative_url().unwrap_or_default(); + + #[cfg(feature="s3")] + if CONFIG.s3.as_ref().map(|x| x.direct_download).unwrap_or(false) { + let s3_url = match CONFIG.s3.as_ref().unwrap() { + S3Config { alias: Some(alias), .. } => { + format!("https://{}/{}", alias, relative_url) + } + S3Config { path_style: true, hostname, bucket, .. } => { + format!("https://{}/{}/{}", + hostname, + bucket, + relative_url + ) + } + S3Config { path_style: false, hostname, bucket, .. } => { + format!("https://{}.{}/{}", + bucket, + hostname, + relative_url + ) + } + }; + return Ok(s3_url); + } + Ok(ap_url(&format!( "{}/{}", Instance::get_local()?.public_domain, - &file_path + relative_url ))) } } @@ -176,9 +243,9 @@ impl Media { #[cfg(feature = "s3")] CONFIG.s3.as_ref().unwrap().get_bucket() - .delete_object_blocking(&self.file_path)?; + .delete_object_blocking(&self.relative_url().ok_or(Error::NotFound)?)?; } else { - fs::remove_file(self.file_path.as_str())?; + fs::remove_file(self.local_path().ok_or(Error::NotFound)?)?; } } diesel::delete(self) @@ -316,12 +383,9 @@ impl Media { } fn determine_mirror_file_path(url: &str) -> PathBuf { - let mut file_path = Path::new(&super::CONFIG.media_directory).join(REMOTE_MEDIA_DIRECTORY); - Url::parse(url) - .map(|url| { - if !url.has_host() { - return; - } + let mut file_path = Path::new(&CONFIG.media_directory).join(REMOTE_MEDIA_DIRECTORY); + match Url::parse(url) { + Ok(url) if url.has_host() => { file_path.push(url.host_str().unwrap()); for segment in url.path_segments().expect("FIXME") { file_path.push(segment); @@ -329,16 +393,21 @@ fn determine_mirror_file_path(url: &str) -> PathBuf { // TODO: handle query // HINT: Use characters which must be percent-encoded in path as separator between path and query // HINT: handle extension - }) - .unwrap_or_else(|err| { - warn!("Failed to parse url: {} {}", &url, err); + } + other => { + if let Err(err) = other { + warn!("Failed to parse url: {} {}", &url, err); + } else { + warn!("Error without a host: {}", &url); + } let ext = url .rsplit('.') .next() .map(ToOwned::to_owned) .unwrap_or_else(|| String::from("png")); file_path.push(format!("{}.{}", GUID::rand(), ext)); - }); + } + } file_path } From 20fa2cacf4372c1820f0dbe5a36c7c160d7e22d0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 15:54:44 +0200 Subject: [PATCH 07/11] Store replicated remote media on S3 if available --- plume-models/src/medias.rs | 93 ++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 15 deletions(-) diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index f371cbab..8c6fecd3 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -287,22 +287,54 @@ impl Media { .url() .and_then(|url| url.to_as_uri()) .ok_or(Error::MissingApProperty)?; - let path = determine_mirror_file_path(&remote_url); - let parent = path.parent().ok_or(Error::InvalidValue)?; - if !parent.is_dir() { - DirBuilder::new().recursive(true).create(parent)?; - } - let mut dest = fs::File::create(path.clone())?; - // TODO: conditional GET - request::get( - remote_url.as_str(), - User::get_sender(), - CONFIG.proxy().cloned(), - )? - .copy_to(&mut dest)?; + let file_path = if CONFIG.s3.is_some() { + #[cfg(not(feature="s3"))] + unreachable!(); - Media::find_by_file_path(conn, path.to_str().ok_or(Error::InvalidValue)?) + #[cfg(feature = "s3")] + { + let dest = determine_mirror_s3_path(&remote_url); + + let media = request::get( + remote_url.as_str(), + User::get_sender(), + CONFIG.proxy().cloned(), + )?; + let content_type = media.headers().get(reqwest::header::CONTENT_TYPE).cloned(); + let bytes = media.bytes()?; + + let bucket = CONFIG.s3.as_ref().unwrap().get_bucket(); + match content_type.as_ref().and_then(|x| x.to_str().ok()) { + Some(ct) => { + bucket.put_object_with_content_type_blocking(&dest, &bytes, ct)?; + } + None => { + bucket.put_object_blocking(&dest, &bytes)?; + } + } + + dest + } + } else { + let path = determine_mirror_file_path(&remote_url); + let parent = path.parent().ok_or(Error::InvalidValue)?; + if !parent.is_dir() { + DirBuilder::new().recursive(true).create(parent)?; + } + + let mut dest = fs::File::create(path.clone())?; + // TODO: conditional GET + request::get( + remote_url.as_str(), + User::get_sender(), + CONFIG.proxy().cloned(), + )? + .copy_to(&mut dest)?; + path.to_str().ok_or(Error::InvalidValue)?.to_string() + }; + + Media::find_by_file_path(conn, &file_path) .and_then(|mut media| { let mut updated = false; @@ -343,7 +375,7 @@ impl Media { Media::insert( conn, NewMedia { - file_path: path.to_str().ok_or(Error::InvalidValue)?.to_string(), + file_path, alt_text: image .content() .and_then(|content| content.to_as_string()) @@ -384,6 +416,7 @@ impl Media { fn determine_mirror_file_path(url: &str) -> PathBuf { let mut file_path = Path::new(&CONFIG.media_directory).join(REMOTE_MEDIA_DIRECTORY); + match Url::parse(url) { Ok(url) if url.has_host() => { file_path.push(url.host_str().unwrap()); @@ -411,6 +444,36 @@ fn determine_mirror_file_path(url: &str) -> PathBuf { file_path } +#[cfg(feature="s3")] +fn determine_mirror_s3_path(url: &str) -> String { + match Url::parse(url) { + Ok(url) if url.has_host() => { + format!("static/media/{}/{}/{}", + REMOTE_MEDIA_DIRECTORY, + url.host_str().unwrap(), + url.path().trim_start_matches('/'), + ) + } + other => { + if let Err(err) = other { + warn!("Failed to parse url: {} {}", &url, err); + } else { + warn!("Error without a host: {}", &url); + } + let ext = url + .rsplit('.') + .next() + .map(ToOwned::to_owned) + .unwrap_or_else(|| String::from("png")); + format!("static/media/{}/{}.{}", + REMOTE_MEDIA_DIRECTORY, + GUID::rand(), + ext, + ) + } + } +} + #[cfg(test)] pub(crate) mod tests { use super::*; From 24d3b289da085261966fb338113610905dfca8c9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 16:11:29 +0200 Subject: [PATCH 08/11] Properly handle Content-Type --- plume-models/src/medias.rs | 24 +++++++++++++++--------- src/routes/medias.rs | 16 ++++++++-------- src/routes/mod.rs | 10 ++++++---- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index 8c6fecd3..8468caf3 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -294,6 +294,8 @@ impl Media { #[cfg(feature = "s3")] { + use rocket::http::ContentType; + let dest = determine_mirror_s3_path(&remote_url); let media = request::get( @@ -301,18 +303,22 @@ impl Media { User::get_sender(), CONFIG.proxy().cloned(), )?; - let content_type = media.headers().get(reqwest::header::CONTENT_TYPE).cloned(); + + let content_type = media + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|x| x.to_str().ok()) + .and_then(ContentType::parse_flexible) + .unwrap_or(ContentType::Binary); + let bytes = media.bytes()?; let bucket = CONFIG.s3.as_ref().unwrap().get_bucket(); - match content_type.as_ref().and_then(|x| x.to_str().ok()) { - Some(ct) => { - bucket.put_object_with_content_type_blocking(&dest, &bytes, ct)?; - } - None => { - bucket.put_object_blocking(&dest, &bytes)?; - } - } + bucket.put_object_with_content_type_blocking( + &dest, + &bytes, + &content_type.to_string() + )?; dest } diff --git a/src/routes/medias.rs b/src/routes/medias.rs index b9a22dee..05119d23 100644 --- a/src/routes/medias.rs +++ b/src/routes/medias.rs @@ -131,14 +131,14 @@ fn save_uploaded_file(file: &SavedField) -> Result, plume_models: }; let bucket = CONFIG.s3.as_ref().unwrap().get_bucket(); - match &file.headers.content_type { - Some(ct) => { - bucket.put_object_with_content_type_blocking(&dest, &bytes, &ct.to_string())?; - } - None => { - bucket.put_object_blocking(&dest, &bytes)?; - } - } + let content_type = match &file.headers.content_type { + Some(ct) => ct.to_string(), + None => ContentType::from_extension(&ext) + .unwrap_or(ContentType::Binary) + .to_string(), + }; + + bucket.put_object_with_content_type_blocking(&dest, &bytes, &content_type)?; Ok(Some(dest)) } diff --git a/src/routes/mod.rs b/src/routes/mod.rs index ac016b9d..5b4adb6f 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -269,13 +269,15 @@ pub fn plume_media_files(file: PathBuf) -> Option { #[cfg(feature="s3")] { - let ct = file.extension() - .and_then(|ext| ContentType::from_extension(&ext.to_string_lossy())) - .unwrap_or(ContentType::Binary); - let data = CONFIG.s3.as_ref().unwrap().get_bucket() .get_object_blocking(format!("static/media/{}", file.to_string_lossy())).ok()?; + let ct = data.headers().get("content-type") + .and_then(|x| ContentType::parse_flexible(&x)) + .or_else(|| file.extension() + .and_then(|ext| ContentType::from_extension(&ext.to_string_lossy()))) + .unwrap_or(ContentType::Binary); + Some(CachedFile { inner: FileKind::S3 ( data.to_vec(), ct), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), From 20d77c22df8afe49d37ada8d6863b90113023146 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 17:20:45 +0200 Subject: [PATCH 09/11] try (and fail) to build with Nix --- flake.lock | 20 +++----------------- flake.nix | 47 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/flake.lock b/flake.lock index b8334cbb..776fb39f 100644 --- a/flake.lock +++ b/flake.lock @@ -52,22 +52,6 @@ "type": "github" } }, - "nixpkgs_2": { - "locked": { - "lastModified": 1681358109, - "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, "root": { "inputs": { "flake-utils": "flake-utils", @@ -78,7 +62,9 @@ "rust-overlay": { "inputs": { "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs_2" + "nixpkgs": [ + "nixpkgs" + ] }, "locked": { "lastModified": 1683857898, diff --git a/flake.nix b/flake.nix index 2646d14c..2563a5b0 100644 --- a/flake.nix +++ b/flake.nix @@ -2,23 +2,58 @@ description = "Developpment shell for Plume including nightly Rust compiler"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - inputs.rust-overlay.url = "github:oxalica/rust-overlay"; + inputs.rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; inputs.flake-utils.url = "github:numtide/flake-utils"; outputs = { self, nixpkgs, flake-utils, rust-overlay, ... }: flake-utils.lib.eachDefaultSystem (system: - let + let overlays = [ (import rust-overlay) ]; pkgs = import nixpkgs { inherit system overlays; }; - in { - devShells.default = pkgs.mkShell { - nativeBuildInputs = with pkgs; [ - rust-bin.nightly.latest.default + inputs = with pkgs; [ + (rust-bin.nightly.latest.default.override { + targets = [ "wasm32-unknown-unknown" ]; + }) + wasm-pack openssl pkg-config gettext postgresql ]; + in { + packages.default = pkgs.rustPlatform.buildRustPackage { + pname = "plume"; + version = "0.7.3-dev"; + + src = ./.; + + cargoLock = { + lockFile = ./Cargo.lock; + outputHashes = { + "pulldown-cmark-0.8.0" = "sha256-lpfoRDuY3zJ3QmUqJ5k9OL0MEdGDpwmpJ+u5BCj2kIA="; + "rocket_csrf-0.1.2" = "sha256-WywZfMiwZqTPfSDcAE7ivTSYSaFX+N9fjnRsLSLb9wE="; + }; + }; + buildNoDefaultFeatures = true; + buildFeatures = ["postgresql" "s3"]; + + nativeBuildInputs = inputs; + + buildPhase = '' + wasm-pack build --target web --release plume-front + cargo build --no-default-features --features postgresql,s3 --path . + cargo build --no-default-features --features postgresql,s3 --path plume-cli + ''; + installPhase = '' + cargo install --no-default-features --features postgresql,s3 --path . --target-dir $out + cargo install --no-default-features --features postgresql,s3 --path plume-cli --target-dir $out + ''; + }; + devShells.default = pkgs.mkShell { + packages = inputs; }; }); } From 3f9321242413e71e15c59613e05730f67a0f77b7 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 12 May 2023 18:25:19 +0200 Subject: [PATCH 10/11] fix plume-cli --- flake.nix | 1 + plume-cli/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 2563a5b0..5ee07fd9 100644 --- a/flake.nix +++ b/flake.nix @@ -22,6 +22,7 @@ pkg-config gettext postgresql + sqlite ]; in { packages.default = pkgs.rustPlatform.buildRustPackage { diff --git a/plume-cli/Cargo.toml b/plume-cli/Cargo.toml index deed153d..236216fd 100644 --- a/plume-cli/Cargo.toml +++ b/plume-cli/Cargo.toml @@ -24,3 +24,4 @@ path = "../plume-models" postgres = ["plume-models/postgres", "diesel/postgres"] sqlite = ["plume-models/sqlite", "diesel/sqlite"] search-lindera = ["plume-models/search-lindera"] +s3 = ["plume-models/s3"] From 61e65a55ad1f5094321c111e395d00dddcb05e96 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 15 May 2023 12:35:39 +0200 Subject: [PATCH 11/11] improve formatting --- plume-models/src/config.rs | 1 + src/routes/medias.rs | 5 ++--- src/routes/mod.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plume-models/src/config.rs b/plume-models/src/config.rs index dc27497b..12441a75 100644 --- a/plume-models/src/config.rs +++ b/plume-models/src/config.rs @@ -369,6 +369,7 @@ pub struct S3Config { pub hostname: String, // may be useful when using self hosted s3. Won't work with recent AWS buckets pub path_style: bool, + // http or https pub protocol: String, // download directly from s3 to user, wihout going through Plume. Require public read on bucket diff --git a/src/routes/medias.rs b/src/routes/medias.rs index 05119d23..5f4b966d 100644 --- a/src/routes/medias.rs +++ b/src/routes/medias.rs @@ -108,7 +108,6 @@ fn save_uploaded_file(file: &SavedField) -> Result, plume_models: Some(ext.to_lowercase()) } }) - .map(|ext| format!(".{}", ext)) }) .unwrap_or_default(); @@ -120,7 +119,7 @@ fn save_uploaded_file(file: &SavedField) -> Result, plume_models: { use std::borrow::Cow; - let dest = format!("static/media/{}{}", GUID::rand(), ext); + let dest = format!("static/media/{}.{}", GUID::rand(), ext); let bytes = match file.data { SavedData::Bytes(ref bytes) => Cow::from(bytes), @@ -143,7 +142,7 @@ fn save_uploaded_file(file: &SavedField) -> Result, plume_models: Ok(Some(dest)) } } else { - let dest = format!("{}/{}{}", CONFIG.media_directory, GUID::rand(), ext); + let dest = format!("{}/{}.{}", CONFIG.media_directory, GUID::rand(), ext); match file.data { SavedData::Bytes(ref bytes) => { diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 5b4adb6f..11c22825 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -279,7 +279,7 @@ pub fn plume_media_files(file: PathBuf) -> Option { .unwrap_or(ContentType::Binary); Some(CachedFile { - inner: FileKind::S3 ( data.to_vec(), ct), + inner: FileKind::S3(data.to_vec(), ct), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), }) }