From 54af93d8fff19d55a55df8ad2ea26e9c31b27419 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sun, 13 Nov 2022 11:18:13 +0100 Subject: [PATCH] initial s3 support probably incomplete --- Cargo.lock | 244 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + plume-models/Cargo.toml | 1 + plume-models/src/config.rs | 108 +++++++++++++++- plume-models/src/lib.rs | 7 ++ plume-models/src/medias.rs | 7 +- src/routes/mod.rs | 37 ++++-- 7 files changed, 388 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a4e162d..a63c579f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,6 +115,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + [[package]] name = "ahash" version = "0.7.6" @@ -166,6 +172,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + [[package]] name = "arc-swap" version = "1.6.0" @@ -227,7 +239,7 @@ dependencies = [ "derive_builder", "diligent-date-parser", "never", - "quick-xml", + "quick-xml 0.27.1", ] [[package]] @@ -241,6 +253,22 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "attohttpc" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e69e13a99a7e6e070bb114f7ff381e58c7ccc188630121fc4c2fe4bcf24cd072" +dependencies = [ + "http 0.2.8", + "log 0.4.17", + "native-tls", + "openssl", + "serde 1.0.152", + "serde_json", + "url 2.3.1", + "wildmatch", +] + [[package]] name = "atty" version = "0.2.14" @@ -267,6 +295,31 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-creds" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460a75eac8f3cb7683e0a9a588a83c3ff039331ea7bfbfbfcecf1dacab276e11" +dependencies = [ + "anyhow", + "attohttpc", + "dirs", + "rust-ini 0.17.0", + "serde 1.0.152", + "serde-xml-rs", + "serde_derive", + "url 2.3.1", +] + +[[package]] +name = "aws-region" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10110ddbd800fb47e6bef95e88fc13495795d252f585272a4fa3ac4f5b2e0a4d" +dependencies = [ + "anyhow", +] + [[package]] name = "backtrace" version = "0.1.8" @@ -389,6 +442,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block_on_proc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b872f3528eeeb4370ee73b51194dc1cd93680c2d0eb6c7a223889038d2c1a167" +dependencies = [ + "quote 1.0.23", + "syn 1.0.107", +] + [[package]] name = "blowfish" version = "0.9.1" @@ -578,7 +641,7 @@ checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", "nom 5.1.2", - "rust-ini", + "rust-ini 0.13.0", "serde 1.0.152", "serde-hjson", "serde_json", @@ -636,7 +699,7 @@ dependencies = [ "aes-gcm", "base64 0.13.1", "hkdf", - "hmac", + "hmac 0.10.1", "percent-encoding 2.2.0", "rand 0.8.5", "sha2", @@ -876,6 +939,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "ctr" version = "0.6.0" @@ -1150,6 +1223,35 @@ dependencies = [ "chrono", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] +name = "dlv-list" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -1745,6 +1847,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash 0.4.7", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -1794,7 +1905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" dependencies = [ "digest", - "hmac", + "hmac 0.10.1", ] [[package]] @@ -1803,7 +1914,17 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ - "crypto-mac", + "crypto-mac 0.10.1", + "digest", +] + +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac 0.11.1", "digest", ] @@ -2549,6 +2670,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2 1.0.49", + "quote 1.0.23", + "syn 1.0.107", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -2641,6 +2773,15 @@ dependencies = [ "unicase 2.6.0", ] +[[package]] +name = "minidom" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" +dependencies = [ + "quick-xml 0.20.0", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3063,6 +3204,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" +dependencies = [ + "dlv-list", + "hashbrown 0.9.1", +] + [[package]] name = "overload" version = "0.1.1" @@ -3294,6 +3445,7 @@ dependencies = [ "rocket_i18n", "rsass", "ructe", + "rust-s3", "scheduled-thread-pool", "serde 1.0.152", "serde_json", @@ -3409,6 +3561,7 @@ dependencies = [ "riker", "rocket", "rocket_i18n", + "rust-s3", "scheduled-thread-pool", "serde 1.0.152", "serde_derive", @@ -3552,6 +3705,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.27.1" @@ -3836,6 +3998,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom 0.2.8", + "redox_syscall 0.2.16", + "thiserror", +] + [[package]] name = "regex" version = "1.7.0" @@ -3967,6 +4140,7 @@ dependencies = [ "tokio 1.24.1", "tokio-native-tls", "tokio-socks", + "tokio-util 0.7.4", "tower-service", "url 2.3.1", "wasm-bindgen", @@ -4159,6 +4333,48 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" +[[package]] +name = "rust-ini" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a4e82923ed07143871571852a390742200607e5058ce633afec89752f9c3f82" +dependencies = [ + "anyhow", + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.1", + "block_on_proc", + "cfg-if 1.0.0", + "hex", + "hmac 0.11.0", + "http 0.2.8", + "log 0.4.17", + "maybe-async", + "md5", + "minidom", + "percent-encoding 2.2.0", + "reqwest 0.11.13", + "serde 1.0.152", + "serde-xml-rs", + "serde_derive", + "sha2", + "time 0.3.17", + "tokio 1.24.1", + "tokio-stream", + "url 2.3.1", +] + [[package]] name = "rust-stemmers" version = "1.2.0" @@ -4302,6 +4518,18 @@ dependencies = [ "serde 0.8.23", ] +[[package]] +name = "serde-xml-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa" +dependencies = [ + "log 0.4.17", + "serde 1.0.152", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.152" @@ -5679,6 +5907,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "wildmatch" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee583bdc5ff1cf9db20e9db5bb3ff4c3089a8f6b8b31aff265c9aba85812db86" + [[package]] name = "winapi" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index 2c2e1d30..693c5d0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ rocket = "0.4.11" rocket_contrib = { version = "0.4.11", features = ["json"] } rocket_i18n = "0.4.1" scheduled-thread-pool = "0.2.6" +rust-s3 = { version = "0.29.0", no-default-features = true, features = ["blocking"], optional = true} serde = "1.0.137" serde_json = "1.0.81" shrinkwraprs = "0.3.0" @@ -74,6 +75,7 @@ sqlite = ["plume-models/sqlite", "diesel/sqlite"] debug-mailer = [] test = [] search-lindera = ["plume-models/search-lindera"] +s3 = ["rust-s3"] [workspace] members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"] diff --git a/plume-models/Cargo.toml b/plume-models/Cargo.toml index 914bf803..c28a6708 100644 --- a/plume-models/Cargo.toml +++ b/plume-models/Cargo.toml @@ -18,6 +18,7 @@ rocket_i18n = "0.4.1" reqwest = "0.11.11" scheduled-thread-pool = "0.2.6" serde = "1.0.137" +rust-s3 = { version = "0.29.0", no-default-features = true, features = ["blocking"] } serde_derive = "1.0" serde_json = "1.0.81" tantivy = "0.13.3" diff --git a/plume-models/src/config.rs b/plume-models/src/config.rs index ba705d42..d9ba0ea7 100644 --- a/plume-models/src/config.rs +++ b/plume-models/src/config.rs @@ -6,6 +6,10 @@ use rocket::Config as RocketConfig; use std::collections::HashSet; use std::env::{self, var}; +use s3::{Bucket, Region}; +use s3::creds::Credentials; + + #[cfg(not(test))] const DB_NAME: &str = "plume"; #[cfg(test)] @@ -27,13 +31,23 @@ pub struct Config { pub mail: Option, pub ldap: Option, pub proxy: Option, + pub s3: Option, } + impl Config { pub fn proxy(&self) -> Option<&reqwest::Proxy> { self.proxy.as_ref().map(|p| &p.proxy) } } +fn string_to_bool(val: &str, name: &str) -> bool { + match val { + "1" | "true" | "TRUE" => true, + "0" | "false" | "FALSE" => false, + _ => panic!("Invalid configuration: {} is not boolean", name), + } +} + #[derive(Debug, Clone)] pub enum InvalidRocketConfig { Env, @@ -288,11 +302,7 @@ fn get_ldap_config() -> Option { match (addr, base_dn) { (Some(addr), Some(base_dn)) => { let tls = var("LDAP_TLS").unwrap_or_else(|_| "false".to_owned()); - let tls = match tls.as_ref() { - "1" | "true" | "TRUE" => true, - "0" | "false" | "FALSE" => false, - _ => panic!("Invalid LDAP configuration : tls"), - }; + let tls = string_to_bool(&tls, "LDAP_TLS"); let user_name_attr = var("LDAP_USER_NAME_ATTR").unwrap_or_else(|_| "cn".to_owned()); let mail_attr = var("LDAP_USER_MAIL_ATTR").unwrap_or_else(|_| "mail".to_owned()); Some(LdapConfig { @@ -349,6 +359,93 @@ fn get_proxy_config() -> Option { }) } +pub struct S3Config { + pub bucket: String, + pub access_key_id: String, + pub access_key_secret: String, + + // region? If not set, default to us-east-1 + pub region: String, + // hostname for s3. If not set, default to $region.amazonaws.com + pub hostname: String, + // may be useful when using self hosted s3. Won't work with recent AWS buckets + pub path_style: bool, + pub protocol: String, + + // options below this comment are not used yet + // upload directly from user to S3, without going through Plume. Uses PostObject endpoint + pub direct_upload: bool, + // download directly from s3 to user, wihout going through Plume. Require public read on bucket + pub direct_download: bool, + // use this hostname for downloads, can be used with caching proxy in front of s3 + pub alias: Option, +} + +impl S3Config { + pub fn get_bucket(&self) -> Bucket { + let region = Region::Custom { + region: self.region.clone(), + endpoint: format!("{}://{}", self.protocol, self.hostname), + }; + let credentials = Credentials { + access_key: Some(self.access_key_id.clone()), + secret_key: Some(self.access_key_secret.clone()), + security_token: None, + session_token: None, + }; + + if self.path_style { + Bucket::new_with_path_style(&self.bucket, region, credentials) + } else { + Bucket::new(&self.bucket, region, credentials) + }.unwrap() + } +} + +fn get_s3_config() -> Option { + let bucket = var("S3_BUCKET").ok(); + let access_key_id = var("AWS_ACCESS_KEY_ID").ok(); + let access_key_secret = var("AWS_SECRET_ACCESS_KEY").ok(); + if bucket.is_none() && access_key_id.is_none() && access_key_secret.is_none() { + return None; + } + if bucket.is_none() || access_key_id.is_none() || access_key_secret.is_none() { + panic!("Invalid S3 configuration: some required values are set, but not others"); + } + let bucket = bucket.unwrap(); + let access_key_id = access_key_id.unwrap(); + let access_key_secret = access_key_secret.unwrap(); + + let region = var("S3_REGION").unwrap_or_else(|_| "us-east-1".to_owned()); + let hostname = var("S3_HOSTNAME").unwrap_or_else(|_| format!("{}.amazonaws.com", region)); + + let protocol = var("S3_PROTOCOL").unwrap_or_else(|_| "https".to_owned()); + if protocol != "http" && protocol != "https" { + panic!("Invalid S3 configuration: invalid protocol {}", protocol); + } + + let path_style = var("S3_PATH_STYLE").unwrap_or_else(|_| "false".to_owned()); + let path_style = string_to_bool(&path_style, "S3_PATH_STYLE"); + let direct_upload = var("S3_DIRECT_UPLOAD").unwrap_or_else(|_| "false".to_owned()); + let direct_upload = string_to_bool(&direct_upload, "S3_DIRECT_UPLOAD"); + let direct_download = var("S3_DIRECT_DOWNLOAD").unwrap_or_else(|_| "false".to_owned()); + let direct_download = string_to_bool(&direct_download, "S3_DIRECT_DOWNLOAD"); + + let alias = var("S3_ALIAS_HOST").ok(); + Some(S3Config { + bucket, + access_key_id, + access_key_secret, + region, + hostname, + protocol, + path_style, + direct_upload, + direct_download, + alias, + }) +} + lazy_static! { pub static ref CONFIG: Config = Config { base_url: var("BASE_URL").unwrap_or_else(|_| format!( @@ -380,5 +477,6 @@ lazy_static! { mail: get_mail_config(), ldap: get_ldap_config(), proxy: get_proxy_config(), + s3: get_s3_config(), }; } diff --git a/plume-models/src/lib.rs b/plume-models/src/lib.rs index e74a201c..b3fa19d9 100644 --- a/plume-models/src/lib.rs +++ b/plume-models/src/lib.rs @@ -69,6 +69,7 @@ pub enum Error { Webfinger, Expired, UserAlreadyExists, + Anyhow(anyhow::Error), } impl From for Error { @@ -170,6 +171,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: anyhow::Error) -> Error { + Error::Anyhow(err) + } +} + pub type Result = std::result::Result; /// Adds a function to a model, that returns the first diff --git a/plume-models/src/medias.rs b/plume-models/src/medias.rs index d3474a2b..192e60da 100644 --- a/plume-models/src/medias.rs +++ b/plume-models/src/medias.rs @@ -170,7 +170,12 @@ impl Media { pub fn delete(&self, conn: &Connection) -> Result<()> { if !self.is_remote { - fs::remove_file(self.file_path.as_str())?; + if let Some(config) = &CONFIG.s3 { + config.get_bucket() + .delete_object_blocking(&self.file_path)?; + } else { + fs::remove_file(self.file_path.as_str())?; + } } diesel::delete(self) .execute(conn) diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 58d71aa6..85fb4b89 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -9,7 +9,7 @@ use rocket::{ http::{ hyper::header::{CacheControl, CacheDirective, ETag, EntityTag}, uri::{FromUriParam, Query}, - RawStr, Status, + ContentType, RawStr, Status, }, request::{self, FromFormValue, FromRequest, Request}, response::{self, Flash, NamedFile, Redirect, Responder, Response}, @@ -204,10 +204,16 @@ pub mod timelines; pub mod user; pub mod well_known; +#[derive(Responder)] +enum FileKind { + Local(NamedFile), + S3(Vec, ContentType), +} + #[derive(Responder)] #[response()] pub struct CachedFile { - inner: NamedFile, + inner: FileKind, cache_control: CacheControl, } @@ -253,19 +259,36 @@ pub fn plume_static_files(file: PathBuf, build_id: &RawStr) -> Option")] pub fn plume_media_files(file: PathBuf) -> Option { - NamedFile::open(Path::new(&CONFIG.media_directory).join(file)) - .ok() - .map(|f| CachedFile { - inner: f, + if let Some(config) = &CONFIG.s3 { + let ct = file.extension() + .and_then(|ext| ContentType::from_extension(&ext.to_string_lossy())) + .unwrap_or(ContentType::Binary); + + let (data, code) = config.get_bucket() + .get_object_blocking(format!("plume-media/{}", file.to_string_lossy())).ok()?; + if code != 200 { + return None; + } + + Some(CachedFile { + inner: FileKind::S3 ( data, ct), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), }) + } else { + NamedFile::open(Path::new(&CONFIG.media_directory).join(file)) + .ok() + .map(|f| CachedFile { + inner: FileKind::Local(f), + cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), + }) + } } #[get("/static/", rank = 3)] pub fn static_files(file: PathBuf) -> Option { NamedFile::open(Path::new("static/").join(file)) .ok() .map(|f| CachedFile { - inner: f, + inner: FileKind::Local(f), cache_control: CacheControl(vec![CacheDirective::MaxAge(60 * 60 * 24 * 30)]), }) }