Merge branch 'main' into wordpress-federation

This commit is contained in:
Felix Ableitner 2024-05-06 11:24:09 +02:00
commit c8e39b577c
5 changed files with 287 additions and 117 deletions

316
Cargo.lock generated
View file

@ -17,7 +17,8 @@ checksum = "8f27d075294830fcab6f66e320dab524bc6d048f4a151698e153205559113772"
[[package]]
name = "activitypub_federation"
version = "0.5.5"
source = "git+https://github.com/LemmyNet/activitypub-federation-rust.git?branch=content-type-case-insensitive#226d850836d56d2b93b970b51bb868b9f2fd1b0a"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e54fe65c4a4b57712d8e436f1fb86ff37e10b56f011f4233fbbfa8c669163e9d"
dependencies = [
"activitystreams-kinds",
"actix-web",
@ -420,47 +421,48 @@ dependencies = [
[[package]]
name = "anstream"
version = "0.6.13"
version = "0.6.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb"
checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"is_terminal_polyfill",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.6"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
[[package]]
name = "anstyle-parse"
version = "0.2.3"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.0.2"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.2"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
dependencies = [
"anstyle",
"windows-sys 0.52.0",
@ -573,6 +575,33 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80"
[[package]]
name = "aws-lc-rs"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5509d663b2c00ee421bda8d6a24d6c42e15970957de1701b8df9f6fbe5707df1"
dependencies = [
"aws-lc-sys",
"mirai-annotations",
"paste",
"zeroize",
]
[[package]]
name = "aws-lc-sys"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d5d317212c2a78d86ba6622e969413c38847b62f48111f8b763af3dac2f9840"
dependencies = [
"bindgen",
"cc",
"cmake",
"dunce",
"fs_extra",
"libc",
"paste",
]
[[package]]
name = "axum"
version = "0.6.20"
@ -714,6 +743,29 @@ dependencies = [
"serde",
]
[[package]]
name = "bindgen"
version = "0.69.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
dependencies = [
"bitflags 2.5.0",
"cexpr",
"clang-sys",
"itertools 0.12.1",
"lazy_static",
"lazycell",
"log",
"prettyplease",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"syn 2.0.60",
"which",
]
[[package]]
name = "bit-set"
version = "0.5.3"
@ -854,6 +906,15 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -895,6 +956,17 @@ dependencies = [
"inout",
]
[[package]]
name = "clang-sys"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "clap"
version = "4.5.4"
@ -944,6 +1016,15 @@ dependencies = [
"chrono",
]
[[package]]
name = "cmake"
version = "0.1.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
dependencies = [
"cc",
]
[[package]]
name = "color-eyre"
version = "0.6.3"
@ -979,9 +1060,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "colorchoice"
version = "1.0.0"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
[[package]]
name = "combine"
@ -1629,6 +1710,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
[[package]]
name = "dunce"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b"
[[package]]
name = "dyn-clone"
version = "1.0.17"
@ -1666,70 +1753,6 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2153bd83ebc09db15bcbdc3e2194d901804952e3dc96967e1cd3b0c5c32d112"
[[package]]
name = "encoding"
version = "0.2.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
dependencies = [
"encoding-index-japanese",
"encoding-index-korean",
"encoding-index-simpchinese",
"encoding-index-singlebyte",
"encoding-index-tradchinese",
]
[[package]]
name = "encoding-index-japanese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-korean"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-simpchinese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-singlebyte"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding-index-tradchinese"
version = "1.20141219.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
dependencies = [
"encoding_index_tests",
]
[[package]]
name = "encoding_index_tests"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
[[package]]
name = "encoding_rs"
version = "0.8.34"
@ -1943,6 +1966,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "fs_extra"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "futf"
version = "0.1.5"
@ -2176,6 +2205,15 @@ dependencies = [
"digest",
]
[[package]]
name = "home"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "hostname"
version = "0.4.0"
@ -2523,6 +2561,12 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
[[package]]
name = "is_terminal_polyfill"
version = "1.70.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
[[package]]
name = "itertools"
version = "0.10.5"
@ -2611,6 +2655,12 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "lemmy_api"
version = "0.19.4-beta.6"
@ -2648,7 +2698,7 @@ dependencies = [
"actix-web",
"anyhow",
"chrono",
"encoding",
"encoding_rs",
"enum-map",
"futures",
"getrandom",
@ -2777,7 +2827,7 @@ dependencies = [
"once_cell",
"pretty_assertions",
"regex",
"rustls 0.21.12",
"rustls 0.23.5",
"serde",
"serde_json",
"serde_with",
@ -2786,7 +2836,7 @@ dependencies = [
"strum_macros",
"tokio",
"tokio-postgres",
"tokio-postgres-rustls 0.10.0",
"tokio-postgres-rustls 0.12.0",
"tracing",
"ts-rs",
"typed-builder",
@ -3012,6 +3062,16 @@ version = "0.2.154"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346"
[[package]]
name = "libloading"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19"
dependencies = [
"cfg-if",
"windows-targets 0.52.5",
]
[[package]]
name = "line-wrap"
version = "0.2.0"
@ -3295,6 +3355,12 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "mirai-annotations"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1"
[[package]]
name = "moka"
version = "0.12.7"
@ -4043,6 +4109,16 @@ dependencies = [
"yansi",
]
[[package]]
name = "prettyplease"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ac2cf0f2e4f42b49f5ffd07dae8d746508ef7526c13940e5f524012ae6c6550"
dependencies = [
"proc-macro2",
"syn 2.0.60",
]
[[package]]
name = "proc-macro2"
version = "1.0.81"
@ -4560,6 +4636,12 @@ version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc_version"
version = "0.4.0"
@ -4634,6 +4716,22 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rustls"
version = "0.23.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afabcee0551bd1aa3e18e5adbf2c0544722014b899adb31bd186ec638d3da97e"
dependencies = [
"aws-lc-rs",
"log",
"once_cell",
"ring 0.17.8",
"rustls-pki-types",
"rustls-webpki 0.102.3",
"subtle",
"zeroize",
]
[[package]]
name = "rustls-channel-resolver"
version = "0.2.0"
@ -4685,6 +4783,7 @@ version = "0.102.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf"
dependencies = [
"aws-lc-rs",
"ring 0.17.8",
"rustls-pki-types",
"untrusted 0.9.0",
@ -4962,6 +5061,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "signal-hook-registry"
version = "1.4.2"
@ -5480,20 +5585,6 @@ dependencies = [
"whoami",
]
[[package]]
name = "tokio-postgres-rustls"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f"
dependencies = [
"futures",
"ring 0.16.20",
"rustls 0.21.12",
"tokio",
"tokio-postgres",
"tokio-rustls 0.24.1",
]
[[package]]
name = "tokio-postgres-rustls"
version = "0.11.1"
@ -5509,6 +5600,20 @@ dependencies = [
"x509-certificate",
]
[[package]]
name = "tokio-postgres-rustls"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04fb792ccd6bbcd4bba408eb8a292f70fc4a3589e5d793626f45190e6454b6ab"
dependencies = [
"ring 0.17.8",
"rustls 0.23.5",
"tokio",
"tokio-postgres",
"tokio-rustls 0.26.0",
"x509-certificate",
]
[[package]]
name = "tokio-rustls"
version = "0.23.4"
@ -5541,6 +5646,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
dependencies = [
"rustls 0.23.5",
"rustls-pki-types",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.15"
@ -6340,6 +6456,18 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "which"
version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
dependencies = [
"either",
"home",
"once_cell",
"rustix 0.38.34",
]
[[package]]
name = "whoami"
version = "1.5.1"

View file

@ -99,7 +99,7 @@ lemmy_db_views = { version = "=0.19.4-beta.6", path = "./crates/db_views" }
lemmy_db_views_actor = { version = "=0.19.4-beta.6", path = "./crates/db_views_actor" }
lemmy_db_views_moderator = { version = "=0.19.4-beta.6", path = "./crates/db_views_moderator" }
lemmy_federate = { version = "=0.19.4-beta.6", path = "./crates/federate" }
activitypub_federation = { git = "https://github.com/LemmyNet/activitypub-federation-rust.git", branch = "content-type-case-insensitive", default-features = false, features = [
activitypub_federation = { version = "0.5.5", default-features = false, features = [
"actix-web",
] }
diesel = "2.1.6"
@ -157,10 +157,10 @@ ts-rs = { version = "7.1.1", features = [
"chrono-impl",
"no-serde-warnings",
] }
rustls = { version = "0.21.11", features = ["dangerous_configuration"] }
rustls = { version = "0.23.5", features = ["ring"] }
futures-util = "0.3.30"
tokio-postgres = "0.7.10"
tokio-postgres-rustls = "0.10.0"
tokio-postgres-rustls = "0.12.0"
urlencoding = "2.1.3"
enum-map = "2.7"
moka = { version = "0.12.7", features = ["future"] }

View file

@ -25,7 +25,7 @@ full = [
"lemmy_db_views_moderator/full",
"lemmy_utils/full",
"activitypub_federation",
"encoding",
"encoding_rs",
"reqwest-middleware",
"webpage",
"ts-rs",
@ -69,7 +69,7 @@ mime = { version = "0.3.17", optional = true }
webpage = { version = "1.6", default-features = false, features = [
"serde",
], optional = true }
encoding = { version = "0.2.33", optional = true }
encoding_rs = { version = "0.8.34", optional = true }
jsonwebtoken = { version = "8.3.0", optional = true }
# necessary for wasmt compilation
getrandom = { version = "0.2.14", features = ["js"] }

View file

@ -6,7 +6,7 @@ use crate::{
utils::{local_site_opt_to_sensitive, proxy_image_link, proxy_image_link_opt_apub},
};
use activitypub_federation::config::Data;
use encoding::{all::encodings, DecoderTrap};
use encoding_rs::{Encoding, UTF_8};
use lemmy_db_schema::{
newtypes::DbUrl,
source::{
@ -160,11 +160,9 @@ fn extract_opengraph_data(html_bytes: &[u8], url: &Url) -> LemmyResult<OpenGraph
// proper encoding. If the specified encoding cannot be found, fall back to the original UTF-8
// version.
if let Some(charset) = page.meta.get("charset") {
if charset.to_lowercase() != "utf-8" {
if let Some(encoding_ref) = encodings().iter().find(|e| e.name() == charset) {
if let Ok(html_with_encoding) = encoding_ref.decode(html_bytes, DecoderTrap::Replace) {
page = HTML::from_string(html_with_encoding, None)?;
}
if charset != UTF_8.name() {
if let Some(encoding) = Encoding::for_label(charset.as_bytes()) {
page = HTML::from_string(encoding.decode(html_bytes).0.into(), None)?;
}
}
}

View file

@ -33,13 +33,22 @@ use lemmy_utils::{
use once_cell::sync::Lazy;
use regex::Regex;
use rustls::{
client::{ServerCertVerified, ServerCertVerifier},
ServerName,
client::danger::{
DangerousClientConfigBuilder,
HandshakeSignatureValid,
ServerCertVerified,
ServerCertVerifier,
},
crypto::{self, verify_tls12_signature, verify_tls13_signature},
pki_types::{CertificateDer, ServerName, UnixTime},
ClientConfig,
DigitallySignedStruct,
SignatureScheme,
};
use std::{
ops::{Deref, DerefMut},
sync::Arc,
time::{Duration, SystemTime},
time::Duration,
};
use tracing::error;
use url::Url;
@ -312,10 +321,11 @@ pub fn diesel_option_overwrite_to_url_create(opt: &Option<String>) -> LemmyResul
fn establish_connection(config: &str) -> BoxFuture<ConnectionResult<AsyncPgConnection>> {
let fut = async {
let rustls_config = rustls::ClientConfig::builder()
.with_safe_defaults()
.with_custom_certificate_verifier(Arc::new(NoCertVerifier {}))
.with_no_client_auth();
let rustls_config = DangerousClientConfigBuilder {
cfg: ClientConfig::builder(),
}
.with_custom_certificate_verifier(Arc::new(NoCertVerifier {}))
.with_no_client_auth();
let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config);
let (client, conn) = tokio_postgres::connect(config, tls)
@ -338,21 +348,55 @@ fn establish_connection(config: &str) -> BoxFuture<ConnectionResult<AsyncPgConne
fut.boxed()
}
#[derive(Debug)]
struct NoCertVerifier {}
impl ServerCertVerifier for NoCertVerifier {
fn verify_server_cert(
&self,
_end_entity: &rustls::Certificate,
_intermediates: &[rustls::Certificate],
_end_entity: &CertificateDer,
_intermediates: &[CertificateDer],
_server_name: &ServerName,
_scts: &mut dyn Iterator<Item = &[u8]>,
_ocsp_response: &[u8],
_now: SystemTime,
_ocsp: &[u8],
_now: UnixTime,
) -> Result<ServerCertVerified, rustls::Error> {
// Will verify all (even invalid) certs without any checks (sslmode=require)
Ok(ServerCertVerified::assertion())
}
fn verify_tls12_signature(
&self,
message: &[u8],
cert: &CertificateDer,
dss: &DigitallySignedStruct,
) -> Result<HandshakeSignatureValid, rustls::Error> {
verify_tls12_signature(
message,
cert,
dss,
&crypto::ring::default_provider().signature_verification_algorithms,
)
}
fn verify_tls13_signature(
&self,
message: &[u8],
cert: &CertificateDer,
dss: &DigitallySignedStruct,
) -> Result<HandshakeSignatureValid, rustls::Error> {
verify_tls13_signature(
message,
cert,
dss,
&crypto::ring::default_provider().signature_verification_algorithms,
)
}
fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
crypto::ring::default_provider()
.signature_verification_algorithms
.supported_schemes()
}
}
pub async fn build_db_pool() -> LemmyResult<ActualDbPool> {