mirrors/lemmy
1
0
Fork 0
mirror of https://github.com/LemmyNet/lemmy.git synced 2024-04-27 07:38:43 +00:00
Code Issues Projects Releases Wiki Activity

Rewrite images to use local proxy (#4035)

Browse source 
* Add markdown rule to add rel=nofollow for all links

* Add markdown image rule to add local image proxy (fixes #1036)

* comments

* rewrite markdown image links working

* add comment

* perform markdown image processing in api/apub receivers

* clippy

* add db table to validate proxied links

* rewrite link fields for avatar, banner etc

* sql fmt

* proxy links received over federation

* add config option

* undo post.url rewriting, move http route definition

* add tests

* proxy images through pictrs

* testing

* cleanup request.rs file

* more cleanup (fixes #2611)

* include url content type when sending post over apub (fixes #2611)

* store post url content type in db

* should be media_type

* get rid of cache_remote_thumbnails setting, instead automatically
take thumbnail from federation data if available.

* fix tests

* add setting disable_external_link_previews

* federate post url as image depending on mime type

* change setting again

* machete

* invert

* support custom emoji

* clippy

* update defaults

* add image proxy test, fix test

* fix test

* clippy

* revert accidental changes

* address review

* clippy

* Markdown link rule-dess (#4356)

* Extracting opengraph_data to its own type.

* A few additions for markdown-link-rule.

---------

Co-authored-by: Nutomic <me@nutomic.com>

* fix setting

* use enum for image proxy setting

* fix test configs

* add config backwards compat

* clippy

* machete

---------

Co-authored-by: Dessalines <dessalines@users.noreply.github.com>
This commit is contained in:
Nutomic 2024-01-25 15:22:11 +01:00 committed by GitHub
parent 1782aafd10
commit e8a52d3a5c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
64 changed files with 1455 additions and 695 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
Cargo.lock generated
View file

@ -2571,6 +2571,8 @@ version = "0.19.3"
dependencies = [
"activitypub_federation",
"actix-web",
"anyhow",
"async-trait",
"chrono",
"encoding",
"enum-map",
@ -2582,8 +2584,8 @@ dependencies = [
"lemmy_db_views_actor",
"lemmy_db_views_moderator",
"lemmy_utils",
"mime",
"once_cell",
"percent-encoding",
"pretty_assertions",
"regex",
"reqwest",
@ -2592,10 +2594,12 @@ dependencies = [
"serde",
"serde_with",
"serial_test",
"task-local-extensions",
"tokio",
"tracing",
"ts-rs",
"url",
"urlencoding",
"uuid",
"webpage",
]
@ -2648,14 +2652,12 @@ dependencies = [
"once_cell",
"pretty_assertions",
"reqwest",
"reqwest-middleware",
"serde",
"serde_json",
"serde_with",
"serial_test",
"stringreader",
"strum_macros",
"task-local-extensions",
"tokio",
"tracing",
"url",
@ -2811,6 +2813,7 @@ dependencies = [
"tokio",
"tracing",
"url",
"urlencoding",
]
[[package]]
@ -2874,7 +2877,6 @@ dependencies = [
"markdown-it",
"once_cell",
"openssl",
"percent-encoding",
"pretty_assertions",
"regex",
"reqwest",
@ -2891,6 +2893,7 @@ dependencies = [
"tracing-error",
"ts-rs",
"url",
"urlencoding",
"uuid",
]

2
Cargo.toml
View file

@ -146,7 +146,6 @@ strum_macros = "0.25.3"
itertools = "0.12.0"
futures = "0.3.30"
http = "0.2.11"
percent-encoding = "2.3.1"
rosetta-i18n = "0.1.3"
opentelemetry = { version = "0.19.0", features = ["rt-tokio"] }
tracing-opentelemetry = { version = "0.19.0" }
@ -155,6 +154,7 @@ rustls = { version = "0.21.10", features = ["dangerous_configuration"] }
futures-util = "0.3.30"
tokio-postgres = "0.7.10"
tokio-postgres-rustls = "0.10.0"
urlencoding = "2.1.3"
enum-map = "2.7"
moka = { version = "0.12.4", features = ["future"] }
i-love-jesus = { version = "0.1.0" }

92
api_tests/src/image.spec.ts
View file

@ -7,16 +7,23 @@ import {
PurgePost,
} from "lemmy-js-client";
import {
alpha,
alphaImage,
alphaUrl,
beta,
betaUrl,
createCommunity,
createPost,
delta,
epsilon,
gamma,
getSite,
registerUser,
resolveBetaCommunity,
resolvePost,
setupLogins,
unfollowRemotes,
waitForPost,
} from "./shared";
const downloadFileSync = require("download-file-sync");
@ -29,9 +36,8 @@ afterAll(() => {
test("Upload image and delete it", async () => {
// Upload test image. We use a simple string buffer as pictrs doesnt require an actual image
// in testing mode.
const upload_image = Buffer.from("test");
const upload_form: UploadImage = {
image: upload_image,
image: Buffer.from("test"),
};
const upload = await alphaImage.uploadImage(upload_form);
expect(upload.files![0].file).toBeDefined();
@ -60,9 +66,8 @@ test("Purge user, uploaded image removed", async () => {
let user = await registerUser(alphaImage, alphaUrl);
// upload test image
const upload_image = Buffer.from("test");
const upload_form: UploadImage = {
image: upload_image,
image: Buffer.from("test"),
};
const upload = await user.uploadImage(upload_form);
expect(upload.files![0].file).toBeDefined();
@ -91,9 +96,8 @@ test("Purge post, linked image removed", async () => {
let user = await registerUser(beta, betaUrl);
// upload test image
const upload_image = Buffer.from("test");
const upload_form: UploadImage = {
image: upload_image,
image: Buffer.from("test"),
};
const upload = await user.uploadImage(upload_form);
expect(upload.files![0].file).toBeDefined();
@ -124,3 +128,79 @@ test("Purge post, linked image removed", async () => {
const content2 = downloadFileSync(upload.url);
expect(content2).toBe("");
});
test("Images in remote post are proxied if setting enabled", async () => {
let user = await registerUser(beta, betaUrl);
let community = await createCommunity(gamma);
const upload_form: UploadImage = {
image: Buffer.from("test"),
};
const upload = await user.uploadImage(upload_form);
let post = await createPost(
gamma,
community.community_view.community.id,
upload.url,
"![](http://example.com/image2.png)",
);
expect(post.post_view.post).toBeDefined();
// remote image gets proxied after upload
expect(
post.post_view.post.url?.startsWith(
"http://lemmy-gamma:8561/api/v3/image_proxy?url",
),
).toBeTruthy();
expect(
post.post_view.post.body?.startsWith(
"![](http://lemmy-gamma:8561/api/v3/image_proxy?url",
),
).toBeTruthy();
let epsilonPost = await resolvePost(epsilon, post.post_view.post);
expect(epsilonPost.post).toBeDefined();
// remote image gets proxied after federation
expect(
epsilonPost.post!.post.url?.startsWith(
"http://lemmy-epsilon:8581/api/v3/image_proxy?url",
),
).toBeTruthy();
expect(
epsilonPost.post!.post.body?.startsWith(
"![](http://lemmy-epsilon:8581/api/v3/image_proxy?url",
),
).toBeTruthy();
});
test("No image proxying if setting is disabled", async () => {
let user = await registerUser(beta, betaUrl);
let community = await createCommunity(alpha);
const upload_form: UploadImage = {
image: Buffer.from("test"),
};
const upload = await user.uploadImage(upload_form);
let post = await createPost(
alpha,
community.community_view.community.id,
upload.url,
"![](http://example.com/image2.png)",
);
expect(post.post_view.post).toBeDefined();
// remote image doesnt get proxied after upload
expect(
post.post_view.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"),
).toBeTruthy();
expect(post.post_view.post.body).toBe("![](http://example.com/image2.png)");
let gammaPost = await resolvePost(delta, post.post_view.post);
expect(gammaPost.post).toBeDefined();
// remote image doesnt get proxied after federation
expect(
gammaPost.post!.post.url?.startsWith("http://127.0.0.1:8551/pictrs/image/"),
).toBeTruthy();
expect(gammaPost.post!.post.body).toBe("![](http://example.com/image2.png)");
});

18
api_tests/src/post.spec.ts
View file

@ -39,7 +39,7 @@ import {
loginUser,
} from "./shared";
import { PostView } from "lemmy-js-client/dist/types/PostView";
import { ResolveObject } from "lemmy-js-client";
import { EditSite, ResolveObject } from "lemmy-js-client";
let betaCommunity: CommunityView | undefined;
@ -72,6 +72,16 @@ function assertPostFederation(postOne?: PostView, postTwo?: PostView) {
}
test("Create a post", async () => {
// Setup some allowlists and blocklists
let editSiteForm: EditSite = {
allowed_instances: ["lemmy-beta"],
};
await delta.editSite(editSiteForm);
editSiteForm.allowed_instances = [];
editSiteForm.blocked_instances = ["lemmy-alpha"];
await epsilon.editSite(editSiteForm);
if (!betaCommunity) {
throw "Missing beta community";
}
@ -109,6 +119,12 @@ test("Create a post", async () => {
await expect(
resolvePost(epsilon, postRes.post_view.post),
).rejects.toStrictEqual(Error("couldnt_find_object"));
// remove added allow/blocklists
editSiteForm.allowed_instances = [];
editSiteForm.blocked_instances = [];
await delta.editSite(editSiteForm);
await epsilon.editSite(editSiteForm);
});
test("Create a post in a non-existent community", async () => {

11
api_tests/src/shared.ts
View file

@ -177,13 +177,6 @@ export async function setupLogins() {
];
await gamma.editSite(editSiteForm);
editSiteForm.allowed_instances = ["lemmy-beta"];
await delta.editSite(editSiteForm);
editSiteForm.allowed_instances = [];
editSiteForm.blocked_instances = ["lemmy-alpha"];
await epsilon.editSite(editSiteForm);
// Create the main alpha/beta communities
// Ignore thrown errors of duplicates
try {
@ -203,10 +196,10 @@ export async function createPost(
api: LemmyHttp,
community_id: number,
url: string = "https://example.com/",
body = randomString(10),
// use example.com for consistent title and embed description
name: string = randomString(5),
): Promise<PostResponse> {
let body = randomString(10);
let form: CreatePost = {
name,
url,
@ -528,7 +521,7 @@ export async function likeComment(
export async function createCommunity(
api: LemmyHttp,
name_: string = randomString(5),
name_: string = randomString(10),
): Promise<CommunityResponse> {
let description = "a sample description";
let form: CreateCommunity = {

35
config/defaults.hjson
View file

@ -36,22 +36,41 @@
# Maximum number of active sql connections
pool_size: 30
}
# Settings related to activitypub federation
# Pictrs image server configuration.
pictrs: {
# Address where pictrs is available (for image hosting)
url: "http://localhost:8080/"
# Set a custom pictrs API key. ( Required for deleting images )
api_key: "string"
# By default the thumbnails for external links are stored in pict-rs. This ensures that they
# can be reliably retrieved and can be resized using pict-rs APIs. However it also increases
# storage usage. In case this is disabled, the Opengraph image is directly returned as
# thumbnail.
# Backwards compatibility with 0.18.1. False is equivalent to `image_mode: None`, true is
# equivalent to `image_mode: StoreLinkPreviews`.
#
# In some countries it is forbidden to copy preview images from newspaper articles and only
# hotlinking is allowed. If that is the case for your instance, make sure that this setting is
# disabled.
# To be removed in 0.20
cache_external_link_previews: true
# Specifies how to handle remote images, so that users don't have to connect directly to remote servers.
image_mode:
# Leave images unchanged, don't generate any local thumbnails for post urls. Instead the the
# Opengraph image is directly returned as thumbnail
"None"
# or
# Generate thumbnails for external post urls and store them persistently in pict-rs. This
# ensures that they can be reliably retrieved and can be resized using pict-rs APIs. However
# it also increases storage usage.
#
# This is the default behaviour, and also matches Lemmy 0.18.
"StoreLinkPreviews"
# or
# If enabled, all images from remote domains are rewritten to pass through `/api/v3/image_proxy`,
# including embedded images in markdown. Images are stored temporarily in pict-rs for caching.
# This improves privacy as users don't expose their IP to untrusted servers, and decreases load
# on other servers. However it increases bandwidth use for the local server.
#
# Requires pict-rs 0.5
"ProxyAllImages"
# Timeout for uploading images to pictrs (in seconds)
upload_timeout: 30
}

17
crates/api/src/local_user/save_settings.rs
View file

@ -2,7 +2,12 @@ use actix_web::web::{Data, Json};
use lemmy_api_common::{
context::LemmyContext,
person::SaveUserSettings,
utils::send_verification_email,
utils::{
local_site_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_api,
send_verification_email,
},
SuccessResponse,
};
use lemmy_db_schema::{
@ -12,7 +17,7 @@ use lemmy_db_schema::{
person::{Person, PersonUpdateForm},
},
traits::Crud,
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url},
utils::diesel_option_overwrite,
};
use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{
@ -28,9 +33,11 @@ pub async fn save_user_settings(
) -> Result<Json<SuccessResponse>, LemmyError> {
let site_view = SiteView::read_local(&mut context.pool()).await?;
let avatar = diesel_option_overwrite_to_url(&data.avatar)?;
let banner = diesel_option_overwrite_to_url(&data.banner)?;
let bio = diesel_option_overwrite(data.bio.clone());
let slur_regex = local_site_to_slur_regex(&site_view.local_site);
let bio = diesel_option_overwrite(process_markdown_opt(&data.bio, &slur_regex, &context).await?);
let avatar = proxy_image_link_opt_api(&data.avatar, &context).await?;
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
let display_name = diesel_option_overwrite(data.display_name.clone());
let matrix_user_id = diesel_option_overwrite(data.matrix_user_id.clone());
let email_deref = data.email.as_deref().map(str::to_lowercase);

4
crates/api/src/post/get_link_metadata.rs
View file

@ -2,7 +2,7 @@ use actix_web::web::{Data, Json, Query};
use lemmy_api_common::{
context::LemmyContext,
post::{GetSiteMetadata, GetSiteMetadataResponse},
request::fetch_site_metadata,
request::fetch_link_metadata,
};
use lemmy_utils::error::LemmyError;
@ -11,7 +11,7 @@ pub async fn get_link_metadata(
data: Query<GetSiteMetadata>,
context: Data<LemmyContext>,
) -> Result<Json<GetSiteMetadataResponse>, LemmyError> {
let metadata = fetch_site_metadata(context.client(), &data.url).await?;
let metadata = fetch_link_metadata(&data.url, false, &context).await?;
Ok(Json(GetSiteMetadataResponse { metadata }))
}

4
crates/api/src/site/purge/person.rs
View file

@ -8,7 +8,7 @@ use lemmy_api_common::{
};
use lemmy_db_schema::{
source::{
image_upload::ImageUpload,
images::LocalImage,
moderator::{AdminPurgePerson, AdminPurgePersonForm},
person::{Person, PersonUpdateForm},
},
@ -31,7 +31,7 @@ pub async fn purge_person(
if let Ok(local_user) = LocalUserView::read_person(&mut context.pool(), person_id).await {
let pictrs_uploads =
ImageUpload::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?;
LocalImage::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?;
for upload in pictrs_uploads {
delete_image_from_pictrs(&upload.pictrs_alias, &upload.pictrs_delete_token, &context)

18
crates/api_common/Cargo.toml
View file

@ -25,7 +25,6 @@ full = [
"lemmy_db_views_actor/full",
"lemmy_db_views_moderator/full",
"activitypub_federation",
"percent-encoding",
"encoding",
"reqwest-middleware",
"webpage",
@ -37,6 +36,7 @@ full = [
"futures",
"once_cell",
"jsonwebtoken",
"mime",
]
[dependencies]
@ -54,11 +54,7 @@ tracing = { workspace = true, optional = true }
reqwest-middleware = { workspace = true, optional = true }
regex = { workspace = true }
rosetta-i18n = { workspace = true, optional = true }
percent-encoding = { workspace = true, optional = true }
webpage = { version = "1.6", default-features = false, features = [
"serde",
], optional = true }
encoding = { version = "0.2.33", optional = true }
anyhow = { workspace = true }
futures = { workspace = true, optional = true }
uuid = { workspace = true, optional = true }
tokio = { workspace = true, optional = true }
@ -66,10 +62,18 @@ reqwest = { workspace = true, optional = true }
ts-rs = { workspace = true, optional = true }
once_cell = { workspace = true, optional = true }
actix-web = { workspace = true, optional = true }
enum-map = { workspace = true }
urlencoding = { workspace = true }
async-trait = { workspace = true }
mime = { version = "0.3.17", optional = true }
webpage = { version = "1.6", default-features = false, features = [
"serde",
], optional = true }
encoding = { version = "0.2.33", optional = true }
jsonwebtoken = { version = "8.3.0", optional = true }
# necessary for wasmt compilation
getrandom = { version = "0.2.12", features = ["js"] }
enum-map = { workspace = true }
task-local-extensions = "0.1.4"
[package.metadata.cargo-machete]
ignored = ["getrandom"]

67
crates/api_common/src/context.rs
View file

@ -1,13 +1,18 @@
use crate::request::client_builder;
use activitypub_federation::config::{Data, FederationConfig};
use anyhow::anyhow;
use lemmy_db_schema::{
source::secret::Secret,
utils::{ActualDbPool, DbPool},
utils::{build_db_pool_for_tests, ActualDbPool, DbPool},
};
use lemmy_utils::{
rate_limit::RateLimitCell,
settings::{structs::Settings, SETTINGS},
};
use reqwest_middleware::ClientWithMiddleware;
use reqwest::{Request, Response};
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, Middleware, Next};
use std::sync::Arc;
use task_local_extensions::Extensions;
#[derive(Clone)]
pub struct LemmyContext {
@ -49,4 +54,62 @@ impl LemmyContext {
pub fn rate_limit_cell(&self) -> &RateLimitCell {
&self.rate_limit_cell
}
/// Initialize a context for use in tests, optionally blocks network requests.
///
/// Do not use this in production code.
pub async fn init_test_context() -> Data<LemmyContext> {
Self::build_test_context(true).await
}
/// Initialize a context for use in tests, with network requests allowed.
/// TODO: get rid of this if possible.
///
/// Do not use this in production code.
pub async fn init_test_context_with_networking() -> Data<LemmyContext> {
Self::build_test_context(false).await
}
async fn build_test_context(block_networking: bool) -> Data<LemmyContext> {
// call this to run migrations
let pool = build_db_pool_for_tests().await;
let client = client_builder(&SETTINGS).build().expect("build client");
let mut client = ClientBuilder::new(client);
if block_networking {
client = client.with(BlockedMiddleware);
}
let client = client.build();
let secret = Secret {
id: 0,
jwt_secret: String::new(),
};
let rate_limit_cell = RateLimitCell::with_test_config();
let context = LemmyContext::create(pool, client, secret, rate_limit_cell.clone());
let config = FederationConfig::builder()
.domain(context.settings().hostname.clone())
.app_data(context)
.build()
.await
.expect("build federation config");
config.to_request_data()
}
}
struct BlockedMiddleware;
/// A reqwest middleware which blocks all requests
#[async_trait::async_trait]
impl Middleware for BlockedMiddleware {
async fn handle(
&self,
_req: Request,
_extensions: &mut Extensions,
_next: Next<'_>,
) -> reqwest_middleware::Result<Response> {
Err(anyhow!("Network requests not allowed").into())
}
}

19
crates/api_common/src/post.rs
View file

@ -238,15 +238,28 @@ pub struct GetSiteMetadata {
#[cfg_attr(feature = "full", ts(export))]
/// The site metadata response.
pub struct GetSiteMetadataResponse {
pub metadata: SiteMetadata,
pub metadata: LinkMetadata,
}
#[skip_serializing_none]
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone)]
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone, Default)]
#[cfg_attr(feature = "full", derive(TS))]
#[cfg_attr(feature = "full", ts(export))]
/// Site metadata, from its opengraph tags.
pub struct SiteMetadata {
pub struct LinkMetadata {
#[serde(flatten)]
pub opengraph_data: OpenGraphData,
pub content_type: Option<String>,
#[serde(skip)]
pub thumbnail: Option<DbUrl>,
}
#[skip_serializing_none]
#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, Clone, Default)]
#[cfg_attr(feature = "full", derive(TS))]
#[cfg_attr(feature = "full", ts(export))]
/// Site metadata, from its opengraph tags.
pub struct OpenGraphData {
pub title: Option<String>,
pub description: Option<String>,
pub(crate) image: Option<DbUrl>,

324
crates/api_common/src/request.rs
View file

@ -1,39 +1,91 @@
use crate::{context::LemmyContext, post::SiteMetadata};
use crate::{
context::LemmyContext,
post::{LinkMetadata, OpenGraphData},
utils::proxy_image_link,
};
use encoding::{all::encodings, DecoderTrap};
use lemmy_db_schema::newtypes::DbUrl;
use lemmy_utils::{
error::{LemmyError, LemmyErrorType},
settings::structs::Settings,
settings::structs::{PictrsImageMode, Settings},
version::VERSION,
REQWEST_TIMEOUT,
};
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::{Client, ClientBuilder};
use mime::Mime;
use reqwest::{header::CONTENT_TYPE, Client, ClientBuilder};
use reqwest_middleware::ClientWithMiddleware;
use serde::Deserialize;
use tracing::info;
use url::Url;
use urlencoding::encode;
use webpage::HTML;
/// Fetches the post link html tags (like title, description, image, etc)
pub fn client_builder(settings: &Settings) -> ClientBuilder {
let user_agent = format!(
"Lemmy/{}; +{}",
VERSION,
settings.get_protocol_and_hostname()
);
Client::builder()
.user_agent(user_agent.clone())
.timeout(REQWEST_TIMEOUT)
.connect_timeout(REQWEST_TIMEOUT)
}
/// Fetches metadata for the given link and optionally generates thumbnail.
#[tracing::instrument(skip_all)]
pub async fn fetch_site_metadata(
client: &ClientWithMiddleware,
pub async fn fetch_link_metadata(
url: &Url,
) -> Result<SiteMetadata, LemmyError> {
generate_thumbnail: bool,
context: &LemmyContext,
) -> Result<LinkMetadata, LemmyError> {
info!("Fetching site metadata for url: {}", url);
let response = client.get(url.as_str()).send().await?;
let response = context.client().get(url.as_str()).send().await?;
let content_type: Option<Mime> = response
.headers()
.get(CONTENT_TYPE)
.and_then(|h| h.to_str().ok())
.and_then(|h| h.parse().ok());
// Can't use .text() here, because it only checks the content header, not the actual bytes
// https://github.com/LemmyNet/lemmy/issues/1964
let html_bytes = response.bytes().await.map_err(LemmyError::from)?.to_vec();
let tags = html_to_site_metadata(&html_bytes, url)?;
let opengraph_data = extract_opengraph_data(&html_bytes, url).unwrap_or_default();
let thumbnail = extract_thumbnail_from_opengraph_data(
url,
&opengraph_data,
&content_type,
generate_thumbnail,
context,
)
.await;
Ok(tags)
Ok(LinkMetadata {
opengraph_data,
content_type: content_type.map(|c| c.to_string()),
thumbnail,
})
}
fn html_to_site_metadata(html_bytes: &[u8], url: &Url) -> Result<SiteMetadata, LemmyError> {
#[tracing::instrument(skip_all)]
pub async fn fetch_link_metadata_opt(
url: Option<&Url>,
generate_thumbnail: bool,
context: &LemmyContext,
) -> LinkMetadata {
match &url {
Some(url) => fetch_link_metadata(url, generate_thumbnail, context)
.await
.unwrap_or_default(),
_ => Default::default(),
}
}
/// Extract site metadata from HTML Opengraph attributes.
fn extract_opengraph_data(html_bytes: &[u8], url: &Url) -> Result<OpenGraphData, LemmyError> {
let html = String::from_utf8_lossy(html_bytes);
// Make sure the first line is doctype html
@ -89,7 +141,7 @@ fn html_to_site_metadata(html_bytes: &[u8], url: &Url) -> Result<SiteMetadata, L
// join also works if the target URL is absolute
.and_then(|v| url.join(&v.url).ok());
Ok(SiteMetadata {
Ok(OpenGraphData {
title: og_title.or(page_title),
description: og_description.or(page_description),
image: og_image.map(Into::into),
@ -97,59 +149,48 @@ fn html_to_site_metadata(html_bytes: &[u8], url: &Url) -> Result<SiteMetadata, L
})
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsResponse {
#[tracing::instrument(skip_all)]
pub async fn extract_thumbnail_from_opengraph_data(
url: &Url,
opengraph_data: &OpenGraphData,
content_type: &Option<Mime>,
generate_thumbnail: bool,
context: &LemmyContext,
) -> Option<DbUrl> {
let is_image = content_type.as_ref().unwrap_or(&mime::TEXT_PLAIN).type_() == mime::IMAGE;
if generate_thumbnail && is_image {
let image_url = opengraph_data
.image
.as_ref()
.map(lemmy_db_schema::newtypes::DbUrl::inner)
.unwrap_or(url);
generate_pictrs_thumbnail(image_url, context)
.await
.ok()
.map(Into::into)
} else {
None
}
}
#[derive(Deserialize, Debug)]
struct PictrsResponse {
files: Vec<PictrsFile>,
msg: String,
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsFile {
#[derive(Deserialize, Debug)]
struct PictrsFile {
file: String,
#[allow(dead_code)]
delete_token: String,
}
#[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsPurgeResponse {
#[derive(Deserialize, Debug)]
struct PictrsPurgeResponse {
msg: String,
}
#[tracing::instrument(skip_all)]
pub(crate) async fn fetch_pictrs(
client: &ClientWithMiddleware,
settings: &Settings,
image_url: &Url,
) -> Result<PictrsResponse, LemmyError> {
let pictrs_config = settings.pictrs_config()?;
is_image_content_type(client, image_url).await?;
if pictrs_config.cache_external_link_previews {
// fetch remote non-pictrs images for persistent thumbnail link
let fetch_url = format!(
"{}image/download?url={}",
pictrs_config.url,
utf8_percent_encode(image_url.as_str(), NON_ALPHANUMERIC) // TODO this might not be needed
);
let response = client
.get(&fetch_url)
.timeout(REQWEST_TIMEOUT)
.send()
.await?;
let response: PictrsResponse = response.json().await.map_err(LemmyError::from)?;
if response.msg == "ok" {
Ok(response)
} else {
Err(LemmyErrorType::PictrsResponseError(response.msg))?
}
} else {
Err(LemmyErrorType::PictrsCachingDisabled)?
}
}
/// Purges an image from pictrs
/// Note: This should often be coerced from a Result to .ok() in order to fail softly, because:
/// - It might fail due to image being not local
@ -167,13 +208,6 @@ pub async fn purge_image_from_pictrs(
.next_back()
.ok_or(LemmyErrorType::ImageUrlMissingLastPathSegment)?;
purge_image_from_pictrs_by_alias(alias, context).await
}
pub async fn purge_image_from_pictrs_by_alias(
alias: &str,
context: &LemmyContext,
) -> Result<(), LemmyError> {
let pictrs_config = context.settings().pictrs_config()?;
let purge_url = format!("{}internal/purge?alias={}", pictrs_config.url, alias);
@ -190,10 +224,9 @@ pub async fn purge_image_from_pictrs_by_alias(
let response: PictrsPurgeResponse = response.json().await.map_err(LemmyError::from)?;
if response.msg == "ok" {
Ok(())
} else {
Err(LemmyErrorType::PictrsPurgeResponseError(response.msg))?
match response.msg.as_str() {
"ok" => Ok(()),
_ => Err(LemmyErrorType::PictrsPurgeResponseError(response.msg))?,
}
}
@ -217,62 +250,48 @@ pub async fn delete_image_from_pictrs(
Ok(())
}
/// Both are options, since the URL might be either an html page, or an image
/// Returns the SiteMetadata, and an image URL, if there is a picture associated
/// Retrieves the image with local pict-rs and generates a thumbnail. Returns the thumbnail url.
#[tracing::instrument(skip_all)]
pub async fn fetch_site_data(
client: &ClientWithMiddleware,
settings: &Settings,
url: Option<&Url>,
include_image: bool,
) -> (Option<SiteMetadata>, Option<DbUrl>) {
match &url {
Some(url) => {
// Fetch metadata
// Ignore errors, since it may be an image, or not have the data.
// Warning, this may ignore SSL errors
let metadata_option = fetch_site_metadata(client, url).await.ok();
if !include_image {
(metadata_option, None)
} else {
let thumbnail_url =
fetch_pictrs_url_from_site_metadata(client, &metadata_option, settings, url)
.await
.ok();
(metadata_option, thumbnail_url)
}
}
None => (None, None),
async fn generate_pictrs_thumbnail(
image_url: &Url,
context: &LemmyContext,
) -> Result<Url, LemmyError> {
let pictrs_config = context.settings().pictrs_config()?;
if pictrs_config.image_mode() == PictrsImageMode::ProxyAllImages {
return Ok(proxy_image_link(image_url.clone(), context).await?.into());
}
// fetch remote non-pictrs images for persistent thumbnail link
// TODO: should limit size once supported by pictrs
let fetch_url = format!(
"{}image/download?url={}",
pictrs_config.url,
encode(image_url.as_str())
);
let response = context
.client()
.get(&fetch_url)
.timeout(REQWEST_TIMEOUT)
.send()
.await?;
let response: PictrsResponse = response.json().await?;
if response.msg == "ok" {
let thumbnail_url = Url::parse(&format!(
"{}/pictrs/image/{}",
context.settings().get_protocol_and_hostname(),
response.files.first().expect("missing pictrs file").file
))?;
Ok(thumbnail_url)
} else {
Err(LemmyErrorType::PictrsResponseError(response.msg))?
}
}
async fn fetch_pictrs_url_from_site_metadata(
client: &ClientWithMiddleware,
metadata_option: &Option<SiteMetadata>,
settings: &Settings,
url: &Url,
) -> Result<DbUrl, LemmyError> {
let pictrs_res = match metadata_option {
Some(metadata_res) => match &metadata_res.image {
// Metadata, with image
// Try to generate a small thumbnail if there's a full sized one from post-links
Some(metadata_image) => fetch_pictrs(client, settings, metadata_image).await,
// Metadata, but no image
None => fetch_pictrs(client, settings, url).await,
},
// No metadata, try to fetch the URL as an image
None => fetch_pictrs(client, settings, url).await,
}?;
Url::parse(&format!(
"{}/pictrs/image/{}",
settings.get_protocol_and_hostname(),
pictrs_res.files.first().expect("missing pictrs file").file
))
.map(Into::into)
.map_err(Into::into)
}
// TODO: get rid of this by reading content type from db
#[tracing::instrument(skip_all)]
async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Result<(), LemmyError> {
let response = client.get(url.as_str()).send().await?;
@ -289,51 +308,50 @@ async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Resu
}
}
pub fn client_builder(settings: &Settings) -> ClientBuilder {
let user_agent = format!(
"Lemmy/{}; +{}",
VERSION,
settings.get_protocol_and_hostname()
);
Client::builder()
.user_agent(user_agent)
.timeout(REQWEST_TIMEOUT)
.connect_timeout(REQWEST_TIMEOUT)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
#![allow(clippy::indexing_slicing)]
use crate::request::{client_builder, fetch_site_metadata, html_to_site_metadata, SiteMetadata};
use lemmy_utils::settings::SETTINGS;
use crate::{
context::LemmyContext,
request::{extract_opengraph_data, fetch_link_metadata},
};
use pretty_assertions::assert_eq;
use serial_test::serial;
use url::Url;
// These helped with testing
#[tokio::test]
async fn test_site_metadata() {
let settings = &SETTINGS.clone();
let client = client_builder(settings).build().unwrap().into();
#[serial]
async fn test_link_metadata() {
let context = LemmyContext::init_test_context_with_networking().await;
let sample_url = Url::parse("https://gitlab.com/IzzyOnDroid/repo/-/wikis/FAQ").unwrap();
let sample_res = fetch_site_metadata(&client, &sample_url).await.unwrap();
let sample_res = fetch_link_metadata(&sample_url, false, &context)
.await
.unwrap();
assert_eq!(
SiteMetadata {
title: Some("FAQ · Wiki · IzzyOnDroid / repo · GitLab".to_string()),
description: Some(
"The F-Droid compatible repo at https://apt.izzysoft.de/fdroid/".to_string()
),
image: Some(
Url::parse("https://gitlab.com/uploads/-/system/project/avatar/4877469/iod_logo.png")
.unwrap()
.into()
),
embed_video_url: None,
},
sample_res
Some("FAQ · Wiki · IzzyOnDroid / repo · GitLab".to_string()),
sample_res.opengraph_data.title
);
assert_eq!(
Some("The F-Droid compatible repo at https://apt.izzysoft.de/fdroid/".to_string()),
sample_res.opengraph_data.description
);
assert_eq!(
Some(
Url::parse("https://gitlab.com/uploads/-/system/project/avatar/4877469/iod_logo.png")
.unwrap()
.into()
),
sample_res.opengraph_data.image
);
assert_eq!(None, sample_res.opengraph_data.embed_video_url);
assert_eq!(
Some(mime::TEXT_HTML_UTF_8.to_string()),
sample_res.content_type
);
assert_eq!(None, sample_res.thumbnail);
}
// #[test]
@ -351,7 +369,7 @@ mod tests {
// root relative url
let html_bytes = b"<!DOCTYPE html><html><head><meta property='og:image' content='/image.jpg'></head><body></body></html>";
let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata");
let metadata = extract_opengraph_data(html_bytes, &url).expect("Unable to parse metadata");
assert_eq!(
metadata.image,
Some(Url::parse("https://example.com/image.jpg").unwrap().into())
@ -359,7 +377,7 @@ mod tests {
// base relative url
let html_bytes = b"<!DOCTYPE html><html><head><meta property='og:image' content='image.jpg'></head><body></body></html>";
let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata");
let metadata = extract_opengraph_data(html_bytes, &url).expect("Unable to parse metadata");
assert_eq!(
metadata.image,
Some(
@ -371,7 +389,7 @@ mod tests {
// absolute url
let html_bytes = b"<!DOCTYPE html><html><head><meta property='og:image' content='https://cdn.host.com/image.jpg'></head><body></body></html>";
let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata");
let metadata = extract_opengraph_data(html_bytes, &url).expect("Unable to parse metadata");
assert_eq!(
metadata.image,
Some(Url::parse("https://cdn.host.com/image.jpg").unwrap().into())
@ -379,7 +397,7 @@ mod tests {
// protocol relative url
let html_bytes = b"<!DOCTYPE html><html><head><meta property='og:image' content='//example.com/image.jpg'></head><body></body></html>";
let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata");
let metadata = extract_opengraph_data(html_bytes, &url).expect("Unable to parse metadata");
assert_eq!(
metadata.image,
Some(Url::parse("https://example.com/image.jpg").unwrap().into())

168
crates/api_common/src/utils.rs
View file

@ -12,6 +12,7 @@ use lemmy_db_schema::{
community::{Community, CommunityModerator, CommunityUpdateForm},
community_block::CommunityBlock,
email_verification::{EmailVerification, EmailVerificationForm},
images::RemoteImage,
instance::Instance,
instance_block::InstanceBlock,
local_site::LocalSite,
@ -35,14 +36,18 @@ use lemmy_utils::{
email::{send_email, translations::Lang},
error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult},
rate_limit::{ActionType, BucketConfig},
settings::structs::Settings,
utils::slurs::build_slur_regex,
settings::structs::{PictrsImageMode, Settings},
utils::{
markdown::markdown_rewrite_image_links,
slurs::{build_slur_regex, remove_slurs},
},
};
use regex::Regex;
use rosetta_i18n::{Language, LanguageId};
use std::collections::HashSet;
use tracing::warn;
use url::{ParseError, Url};
use urlencoding::encode;
pub static AUTH_COOKIE_NAME: &str = "jwt";
@ -848,14 +853,115 @@ fn limit_expire_time(expires: DateTime<Utc>) -> LemmyResult<Option<DateTime<Utc>
}
}
pub async fn process_markdown(
text: &str,
slur_regex: &Option<Regex>,
context: &LemmyContext,
) -> LemmyResult<String> {
let text = remove_slurs(text, slur_regex);
if context.settings().pictrs_config()?.image_mode() == PictrsImageMode::ProxyAllImages {
let (text, links) = markdown_rewrite_image_links(text);
RemoteImage::create(&mut context.pool(), links).await?;
Ok(text)
} else {
Ok(text)
}
}
pub async fn process_markdown_opt(
text: &Option<String>,
slur_regex: &Option<Regex>,
context: &LemmyContext,
) -> LemmyResult<Option<String>> {
match text {
Some(t) => process_markdown(t, slur_regex, context).await.map(Some),
None => Ok(None),
}
}
/// A wrapper for `proxy_image_link` for use in tests.
///
/// The parameter `force_image_proxy` is the config value of `pictrs.image_proxy`. Its necessary to pass
/// as separate parameter so it can be changed in tests.
async fn proxy_image_link_internal(
link: Url,
image_mode: PictrsImageMode,
context: &LemmyContext,
) -> LemmyResult<DbUrl> {
// Dont rewrite links pointing to local domain.
if link.domain() == Some(&context.settings().hostname) {
Ok(link.into())
} else if image_mode == PictrsImageMode::ProxyAllImages {
let proxied = format!(
"{}/api/v3/image_proxy?url={}",
context.settings().get_protocol_and_hostname(),
encode(link.as_str())
);
RemoteImage::create(&mut context.pool(), vec![link]).await?;
Ok(Url::parse(&proxied)?.into())
} else {
Ok(link.into())
}
}
/// Rewrite a link to go through `/api/v3/image_proxy` endpoint. This is only for remote urls and
/// if image_proxy setting is enabled.
pub(crate) async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult<DbUrl> {
proxy_image_link_internal(
link,
context.settings().pictrs_config()?.image_mode(),
context,
)
.await
}
pub async fn proxy_image_link_opt_api(
link: &Option<String>,
context: &LemmyContext,
) -> LemmyResult<Option<Option<DbUrl>>> {
proxy_image_link_api(link, context).await.map(Some)
}
pub async fn proxy_image_link_api(
link: &Option<String>,
context: &LemmyContext,
) -> LemmyResult<Option<DbUrl>> {
let link: Option<DbUrl> = match link.as_ref().map(String::as_str) {
// An empty string is an erase
Some("") => None,
Some(str_url) => Url::parse(str_url)
.map(|u| Some(u.into()))
.with_lemmy_type(LemmyErrorType::InvalidUrl)?,
None => None,
};
if let Some(l) = link {
proxy_image_link(l.into(), context).await.map(Some)
} else {
Ok(link)
}
}
pub async fn proxy_image_link_opt_apub(
link: Option<Url>,
context: &LemmyContext,
) -> LemmyResult<Option<DbUrl>> {
if let Some(l) = link {
proxy_image_link(l, context).await.map(Some)
} else {
Ok(None)
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
#![allow(clippy::indexing_slicing)]
use super::*;
use crate::utils::{honeypot_check, limit_expire_time, password_length_check};
use chrono::{Days, Utc};
use pretty_assertions::assert_eq;
use serial_test::serial;
#[test]
#[rustfmt::skip]
@ -894,4 +1000,62 @@ mod tests {
None
);
}
#[tokio::test]
#[serial]
async fn test_proxy_image_link() {
let context = LemmyContext::init_test_context().await;
// image from local domain is unchanged
let local_url = Url::parse("http://lemmy-alpha/image.png").unwrap();
let proxied =
proxy_image_link_internal(local_url.clone(), PictrsImageMode::ProxyAllImages, &context)
.await
.unwrap();
assert_eq!(&local_url, proxied.inner());
// image from remote domain is proxied
let remote_image = Url::parse("http://lemmy-beta/image.png").unwrap();
let proxied = proxy_image_link_internal(
remote_image.clone(),
PictrsImageMode::ProxyAllImages,
&context,
)
.await
.unwrap();
assert_eq!(
"https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Flemmy-beta%2Fimage.png",
proxied.as_str()
);
assert!(
RemoteImage::validate(&mut context.pool(), remote_image.into())
.await
.is_ok()
);
}
#[tokio::test]
#[serial]
async fn test_diesel_option_overwrite_to_url() {
let context = LemmyContext::init_test_context().await;
assert!(matches!(
proxy_image_link_api(&None, &context).await,
Ok(None)
));
assert!(matches!(
proxy_image_link_opt_api(&Some(String::new()), &context).await,
Ok(Some(None))
));
assert!(
proxy_image_link_opt_api(&Some("invalid_url".to_string()), &context)
.await
.is_err()
);
let example_url = "https://lemmy-alpha/image.png";
assert!(matches!(
proxy_image_link_opt_api(&Some(example_url.to_string()), &context).await,
Ok(Some(Some(url))) if url == Url::parse(example_url).unwrap().into()
));
}
}

13
crates/api_crud/src/comment/create.rs
View file

@ -11,6 +11,7 @@ use lemmy_api_common::{
generate_local_apub_endpoint,
get_post,
local_site_to_slur_regex,
process_markdown,
EndpointType,
},
};
@ -28,11 +29,7 @@ use lemmy_db_schema::{
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::{
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
utils::{
mention::scrape_text_for_mentions,
slurs::remove_slurs,
validation::is_valid_body_field,
},
utils::{mention::scrape_text_for_mentions, validation::is_valid_body_field},
};
const MAX_COMMENT_DEPTH_LIMIT: usize = 100;
@ -45,10 +42,8 @@ pub async fn create_comment(
) -> Result<Json<CommentResponse>, LemmyError> {
let local_site = LocalSite::read(&mut context.pool()).await?;
let content = remove_slurs(
&data.content.clone(),
&local_site_to_slur_regex(&local_site),
);
let slur_regex = local_site_to_slur_regex(&local_site);
let content = process_markdown(&data.content, &slur_regex, &context).await?;
is_valid_body_field(&Some(content.clone()), false)?;
// Check for a community ban

15
crates/api_crud/src/comment/update.rs
View file

@ -5,7 +5,7 @@ use lemmy_api_common::{
comment::{CommentResponse, EditComment},
context::LemmyContext,
send_activity::{ActivityChannel, SendActivityData},
utils::{check_community_user_action, local_site_to_slur_regex},
utils::{check_community_user_action, local_site_to_slur_regex, process_markdown_opt},
};
use lemmy_db_schema::{
source::{
@ -19,11 +19,7 @@ use lemmy_db_schema::{
use lemmy_db_views::structs::{CommentView, LocalUserView};
use lemmy_utils::{
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
utils::{
mention::scrape_text_for_mentions,
slurs::remove_slurs,
validation::is_valid_body_field,
},
utils::{mention::scrape_text_for_mentions, validation::is_valid_body_field},
};
#[tracing::instrument(skip(context))]
@ -57,11 +53,8 @@ pub async fn update_comment(
)
.await?;
// Update the Content
let content = data
.content
.as_ref()
.map(|c| remove_slurs(c, &local_site_to_slur_regex(&local_site)));
let slur_regex = local_site_to_slur_regex(&local_site);
let content = process_markdown_opt(&data.content, &slur_regex, &context).await?;
is_valid_body_field(&content, false)?;
let comment_id = data.comment_id;

15
crates/api_crud/src/community/create.rs
View file

@ -11,6 +11,8 @@ use lemmy_api_common::{
generate_shared_inbox_url,
is_admin,
local_site_to_slur_regex,
process_markdown_opt,
proxy_image_link_api,
EndpointType,
},
};
@ -27,13 +29,12 @@ use lemmy_db_schema::{
},
},
traits::{ApubActor, Crud, Followable, Joinable},
utils::diesel_option_overwrite_to_url_create,
};
use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
utils::{
slurs::{check_slurs, check_slurs_opt},
slurs::check_slurs,
validation::{is_valid_actor_name, is_valid_body_field},
},
};
@ -51,14 +52,12 @@ pub async fn create_community(
Err(LemmyErrorType::OnlyAdminsCanCreateCommunities)?
}
// Check to make sure the icon and banners are urls
let icon = diesel_option_overwrite_to_url_create(&data.icon)?;
let banner = diesel_option_overwrite_to_url_create(&data.banner)?;
let slur_regex = local_site_to_slur_regex(&local_site);
check_slurs(&data.name, &slur_regex)?;
check_slurs(&data.title, &slur_regex)?;
check_slurs_opt(&data.description, &slur_regex)?;
let description = process_markdown_opt(&data.description, &slur_regex, &context).await?;
let icon = proxy_image_link_api(&data.icon, &context).await?;
let banner = proxy_image_link_api(&data.banner, &context).await?;
is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?;
is_valid_body_field(&data.description, false)?;
@ -81,7 +80,7 @@ pub async fn create_community(
let community_form = CommunityInsertForm::builder()
.name(data.name.clone())
.title(data.title.clone())
.description(data.description.clone())
.description(description)
.icon(icon)
.banner(banner)
.nsfw(data.nsfw)

17
crates/api_crud/src/community/update.rs
View file

@ -5,7 +5,12 @@ use lemmy_api_common::{
community::{CommunityResponse, EditCommunity},
context::LemmyContext,
send_activity::{ActivityChannel, SendActivityData},
utils::{check_community_mod_action, local_site_to_slur_regex},
utils::{
check_community_mod_action,
local_site_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_api,
},
};
use lemmy_db_schema::{
source::{
@ -14,7 +19,7 @@ use lemmy_db_schema::{
local_site::LocalSite,
},
traits::Crud,
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
utils::{diesel_option_overwrite, naive_now},
};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::{
@ -32,12 +37,12 @@ pub async fn update_community(
let slur_regex = local_site_to_slur_regex(&local_site);
check_slurs_opt(&data.title, &slur_regex)?;
check_slurs_opt(&data.description, &slur_regex)?;
let description = process_markdown_opt(&data.description, &slur_regex, &context).await?;
is_valid_body_field(&data.description, false)?;
let icon = diesel_option_overwrite_to_url(&data.icon)?;
let banner = diesel_option_overwrite_to_url(&data.banner)?;
let description = diesel_option_overwrite(data.description.clone());
let description = diesel_option_overwrite(description);
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
// Verify its a mod (only mods can edit it)
check_community_mod_action(

29
crates/api_crud/src/post/create.rs
View file

@ -4,7 +4,7 @@ use lemmy_api_common::{
build_response::build_post_response,
context::LemmyContext,
post::{CreatePost, PostResponse},
request::fetch_site_data,
request::fetch_link_metadata_opt,
send_activity::{ActivityChannel, SendActivityData},
utils::{
check_community_user_action,
@ -12,6 +12,8 @@ use lemmy_api_common::{
honeypot_check,
local_site_to_slur_regex,
mark_post_as_read,
process_markdown_opt,
proxy_image_link_opt_apub,
EndpointType,
},
};
@ -31,7 +33,7 @@ use lemmy_utils::{
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
spawn_try_task,
utils::{
slurs::{check_slurs, check_slurs_opt},
slurs::check_slurs,
validation::{check_url_scheme, clean_url_params, is_valid_body_field, is_valid_post_title},
},
};
@ -49,14 +51,14 @@ pub async fn create_post(
let slur_regex = local_site_to_slur_regex(&local_site);
check_slurs(&data.name, &slur_regex)?;
check_slurs_opt(&data.body, &slur_regex)?;
let body = process_markdown_opt(&data.body, &slur_regex, &context).await?;
honeypot_check(&data.honeypot)?;
let data_url = data.url.as_ref();
let url = data_url.map(clean_url_params).map(Into::into); // TODO no good way to handle a "clear"
let url = data_url.map(clean_url_params); // TODO no good way to handle a "clear"
is_valid_post_title(&data.name)?;
is_valid_body_field(&data.body, true)?;
is_valid_body_field(&body, true)?;
check_url_scheme(&data.url)?;
check_community_user_action(
@ -82,11 +84,8 @@ pub async fn create_post(
}
// Fetch post links and pictrs cached image
let (metadata_res, thumbnail_url) =
fetch_site_data(context.client(), context.settings(), data_url, true).await;
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (u.title, u.description, u.embed_video_url))
.unwrap_or_default();
let metadata = fetch_link_metadata_opt(url.as_ref(), true, &context).await;
let url = proxy_image_link_opt_apub(url, &context).await?;
// Only need to check if language is allowed in case user set it explicitly. When using default
// language, it already only returns allowed languages.
@ -113,15 +112,15 @@ pub async fn create_post(
let post_form = PostInsertForm::builder()
.name(data.name.trim().to_string())
.url(url)
.body(data.body.clone())
.body(body)
.community_id(data.community_id)
.creator_id(local_user_view.person.id)
.nsfw(data.nsfw)
.embed_title(embed_title)
.embed_description(embed_description)
.embed_video_url(embed_video_url)
.embed_title(metadata.opengraph_data.title)
.embed_description(metadata.opengraph_data.description)
.embed_video_url(metadata.opengraph_data.embed_video_url)
.language_id(language_id)
.thumbnail_url(thumbnail_url)
.thumbnail_url(metadata.thumbnail)
.build();
let inserted_post = Post::create(&mut context.pool(), &post_form)

45
crates/api_crud/src/post/update.rs
View file

@ -4,9 +4,14 @@ use lemmy_api_common::{
build_response::build_post_response,
context::LemmyContext,
post::{EditPost, PostResponse},
request::fetch_site_data,
request::fetch_link_metadata,
send_activity::{ActivityChannel, SendActivityData},
utils::{check_community_user_action, local_site_to_slur_regex},
utils::{
check_community_user_action,
local_site_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_apub,
},
};
use lemmy_db_schema::{
source::{
@ -35,21 +40,19 @@ pub async fn update_post(
) -> Result<Json<PostResponse>, LemmyError> {
let local_site = LocalSite::read(&mut context.pool()).await?;
let data_url = data.url.as_ref();
// TODO No good way to handle a clear.
// Issue link: https://github.com/LemmyNet/lemmy/issues/2287
let url = Some(data_url.map(clean_url_params).map(Into::into));
let url = data.url.as_ref().map(clean_url_params);
let slur_regex = local_site_to_slur_regex(&local_site);
check_slurs_opt(&data.name, &slur_regex)?;
check_slurs_opt(&data.body, &slur_regex)?;
let body = process_markdown_opt(&data.body, &slur_regex, &context).await?;
if let Some(name) = &data.name {
is_valid_post_title(name)?;
}
is_valid_body_field(&data.body, true)?;
is_valid_body_field(&body, true)?;
check_url_scheme(&data.url)?;
let post_id = data.post_id;
@ -67,13 +70,23 @@ pub async fn update_post(
Err(LemmyErrorType::NoPostEditAllowed)?
}
// Fetch post links and Pictrs cached image
let data_url = data.url.as_ref();
let (metadata_res, thumbnail_url) =
fetch_site_data(context.client(), context.settings(), data_url, true).await;
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
.unwrap_or_default();
// Fetch post links and Pictrs cached image if url was updated
let (embed_title, embed_description, embed_video_url, thumbnail_url) = match &url {
Some(url) => {
let metadata = fetch_link_metadata(url, true, &context).await?;
(
Some(metadata.opengraph_data.title),
Some(metadata.opengraph_data.description),
Some(metadata.opengraph_data.embed_video_url),
Some(metadata.thumbnail),
)
}
_ => Default::default(),
};
let url = match url {
Some(url) => Some(proxy_image_link_opt_apub(Some(url), &context).await?),
_ => Default::default(),
};
let language_id = data.language_id;
CommunityLanguage::is_allowed_community_language(
@ -86,13 +99,13 @@ pub async fn update_post(
let post_form = PostUpdateForm {
name: data.name.clone(),
url,
body: diesel_option_overwrite(data.body.clone()),
body: diesel_option_overwrite(body),
nsfw: data.nsfw,
embed_title,
embed_description,
embed_video_url,
language_id: data.language_id,
thumbnail_url: Some(thumbnail_url),
thumbnail_url,
updated: Some(Some(naive_now())),
..Default::default()
};

6
crates/api_crud/src/private_message/create.rs
View file

@ -9,6 +9,7 @@ use lemmy_api_common::{
generate_local_apub_endpoint,
get_interface_language,
local_site_to_slur_regex,
process_markdown,
send_email_to_user,
EndpointType,
},
@ -23,7 +24,7 @@ use lemmy_db_schema::{
use lemmy_db_views::structs::{LocalUserView, PrivateMessageView};
use lemmy_utils::{
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
utils::{markdown::markdown_to_html, slurs::remove_slurs, validation::is_valid_body_field},
utils::{markdown::markdown_to_html, validation::is_valid_body_field},
};
#[tracing::instrument(skip(context))]
@ -34,7 +35,8 @@ pub async fn create_private_message(
) -> Result<Json<PrivateMessageResponse>, LemmyError> {
let local_site = LocalSite::read(&mut context.pool()).await?;
let content = remove_slurs(&data.content, &local_site_to_slur_regex(&local_site));
let slur_regex = local_site_to_slur_regex(&local_site);
let content = process_markdown(&data.content, &slur_regex, &context).await?;
is_valid_body_field(&Some(content.clone()), false)?;
check_person_block(

7
crates/api_crud/src/private_message/update.rs
View file

@ -4,7 +4,7 @@ use lemmy_api_common::{
context::LemmyContext,
private_message::{EditPrivateMessage, PrivateMessageResponse},
send_activity::{ActivityChannel, SendActivityData},
utils::local_site_to_slur_regex,
utils::{local_site_to_slur_regex, process_markdown},
};
use lemmy_db_schema::{
source::{
@ -17,7 +17,7 @@ use lemmy_db_schema::{
use lemmy_db_views::structs::{LocalUserView, PrivateMessageView};
use lemmy_utils::{
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
utils::{slurs::remove_slurs, validation::is_valid_body_field},
utils::validation::is_valid_body_field,
};
#[tracing::instrument(skip(context))]
@ -36,7 +36,8 @@ pub async fn update_private_message(
}
// Doing the update
let content = remove_slurs(&data.content, &local_site_to_slur_regex(&local_site));
let slur_regex = local_site_to_slur_regex(&local_site);
let content = process_markdown(&data.content, &slur_regex, &context).await?;
is_valid_body_field(&Some(content.clone()), false)?;
let private_message_id = data.private_message_id;

22
crates/api_crud/src/site/create.rs
View file

@ -4,7 +4,14 @@ use actix_web::web::{Data, Json};
use lemmy_api_common::{
context::LemmyContext,
site::{CreateSite, SiteResponse},
utils::{generate_shared_inbox_url, is_admin, local_site_rate_limit_to_rate_limit_config},
utils::{
generate_shared_inbox_url,
is_admin,
local_site_rate_limit_to_rate_limit_config,
local_site_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_api,
},
};
use lemmy_db_schema::{
newtypes::DbUrl,
@ -15,7 +22,7 @@ use lemmy_db_schema::{
tagline::Tagline,
},
traits::Crud,
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
utils::{diesel_option_overwrite, naive_now},
};
use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{
@ -50,12 +57,17 @@ pub async fn create_site(
let inbox_url = Some(generate_shared_inbox_url(context.settings())?);
let keypair = generate_actor_keypair()?;
let slur_regex = local_site_to_slur_regex(&local_site);
let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?;
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
let site_form = SiteUpdateForm {
name: Some(data.name.clone()),
sidebar: diesel_option_overwrite(data.sidebar.clone()),
sidebar: diesel_option_overwrite(sidebar),
description: diesel_option_overwrite(data.description.clone()),
icon: diesel_option_overwrite_to_url(&data.icon)?,
banner: diesel_option_overwrite_to_url(&data.banner)?,
icon,
banner,
actor_id: Some(actor_id),
last_refreshed_at: Some(naive_now()),
inbox_url,

21
crates/api_crud/src/site/update.rs
View file

@ -3,7 +3,13 @@ use actix_web::web::{Data, Json};
use lemmy_api_common::{
context::LemmyContext,
site::{EditSite, SiteResponse},
utils::{is_admin, local_site_rate_limit_to_rate_limit_config},
utils::{
is_admin,
local_site_rate_limit_to_rate_limit_config,
local_site_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_api,
},
};
use lemmy_db_schema::{
source::{
@ -17,7 +23,7 @@ use lemmy_db_schema::{
tagline::Tagline,
},
traits::Crud,
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
utils::{diesel_option_overwrite, naive_now},
RegistrationMode,
};
use lemmy_db_views::structs::{LocalUserView, SiteView};
@ -54,12 +60,17 @@ pub async fn update_site(
SiteLanguage::update(&mut context.pool(), discussion_languages.clone(), &site).await?;
}
let slur_regex = local_site_to_slur_regex(&local_site);
let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &context).await?;
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
let site_form = SiteUpdateForm {
name: data.name.clone(),
sidebar: diesel_option_overwrite(data.sidebar.clone()),
sidebar: diesel_option_overwrite(sidebar),
description: diesel_option_overwrite(data.description.clone()),
icon: diesel_option_overwrite_to_url(&data.icon)?,
banner: diesel_option_overwrite_to_url(&data.banner)?,
icon,
banner,
updated: Some(Some(naive_now())),
..Default::default()
};

2
crates/apub/Cargo.toml
View file

@ -50,7 +50,5 @@ enum_delegate = "0.2.0"
[dev-dependencies]
serial_test = { workspace = true }
reqwest-middleware = { workspace = true }
task-local-extensions = "0.1.4"
assert-json-diff = "2.0.2"
pretty_assertions = { workspace = true }

37
crates/apub/src/activities/community/update.rs
View file

@ -8,7 +8,7 @@ use crate::{
},
activity_lists::AnnouncableActivities,
insert_received_activity,
objects::{community::ApubCommunity, person::ApubPerson},
objects::{community::ApubCommunity, person::ApubPerson, read_from_string_or_source_opt},
protocol::{activities::community::update::UpdateCommunity, InCommunity},
};
use activitypub_federation::{
@ -18,8 +18,13 @@ use activitypub_federation::{
};
use lemmy_api_common::context::LemmyContext;
use lemmy_db_schema::{
source::{activity::ActivitySendTargets, community::Community, person::Person},
source::{
activity::ActivitySendTargets,
community::{Community, CommunityUpdateForm},
person::Person,
},
traits::Crud,
utils::naive_now,
};
use lemmy_utils::error::LemmyError;
use url::Url;
@ -85,7 +90,33 @@ impl ActivityHandler for UpdateCommunity {
insert_received_activity(&self.id, context).await?;
let community = self.community(context).await?;
let community_update_form = self.object.into_update_form();
let community_update_form = CommunityUpdateForm {
title: Some(self.object.name.unwrap_or(self.object.preferred_username)),
description: Some(read_from_string_or_source_opt(
&self.object.summary,
&None,
&self.object.source,
)),
removed: None,
published: self.object.published.map(Into::into),
updated: Some(self.object.updated.map(Into::into)),
deleted: None,
nsfw: Some(self.object.sensitive.unwrap_or(false)),
actor_id: Some(self.object.id.into()),
local: None,
private_key: None,
hidden: None,
public_key: Some(self.object.public_key.public_key_pem),
last_refreshed_at: Some(naive_now()),
icon: Some(self.object.icon.map(|i| i.url.into())),
banner: Some(self.object.image.map(|i| i.url.into())),
followers_url: Some(self.object.followers.into()),
inbox_url: Some(self.object.inbox.into()),
shared_inbox_url: Some(self.object.endpoints.map(|e| e.shared_inbox.into())),
moderators_url: self.object.attributed_to.map(Into::into),
posting_restricted_to_mods: self.object.posting_restricted_to_mods,
featured_url: self.object.featured.map(Into::into),
};
Community::update(&mut context.pool(), community.id, &community_update_form).await?;
Ok(())

9
crates/apub/src/api/user_settings_backup.rs
View file

@ -298,10 +298,7 @@ pub async fn import_settings(
mod tests {
#![allow(clippy::indexing_slicing)]
use crate::{
api::user_settings_backup::{export_settings, import_settings},
objects::tests::init_context,
};
use crate::api::user_settings_backup::{export_settings, import_settings};
use activitypub_federation::config::Data;
use lemmy_api_common::context::LemmyContext;
use lemmy_db_schema::{
@ -348,7 +345,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn test_settings_export_import() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let export_user =
create_user("hanna".to_string(), Some("my bio".to_string()), &context).await?;
@ -397,7 +394,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn disallow_large_backup() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let export_user =
create_user("hanna".to_string(), Some("my bio".to_string()), &context).await?;

8
crates/apub/src/collections/community_moderators.rs
View file

@ -106,11 +106,7 @@ mod tests {
use super::*;
use crate::{
objects::{
community::tests::parse_lemmy_community,
person::tests::parse_lemmy_person,
tests::init_context,
},
objects::{community::tests::parse_lemmy_community, person::tests::parse_lemmy_person},
protocol::tests::file_to_json_object,
};
use lemmy_db_schema::{
@ -129,7 +125,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn test_parse_lemmy_community_moderators() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let (new_mod, site) = parse_lemmy_person(&context).await?;
let community = parse_lemmy_community(&context).await?;
let community_id = community.id;

14
crates/apub/src/objects/comment.rs
View file

@ -16,7 +16,10 @@ use activitypub_federation::{
traits::Object,
};
use chrono::{DateTime, Utc};
use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex};
use lemmy_api_common::{
context::LemmyContext,
utils::{local_site_opt_to_slur_regex, process_markdown},
};
use lemmy_db_schema::{
source::{
comment::{Comment, CommentInsertForm, CommentUpdateForm},
@ -29,7 +32,7 @@ use lemmy_db_schema::{
};
use lemmy_utils::{
error::{LemmyError, LemmyErrorType},
utils::{markdown::markdown_to_html, slurs::remove_slurs},
utils::markdown::markdown_to_html,
};
use std::ops::Deref;
use url::Url;
@ -158,7 +161,7 @@ impl Object for ApubComment {
let local_site = LocalSite::read(&mut context.pool()).await.ok();
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let content = remove_slurs(&content, slur_regex);
let content = process_markdown(&content, slur_regex, context).await?;
let language_id =
LanguageTag::to_language_id_single(note.language, &mut context.pool()).await?;
@ -190,7 +193,6 @@ pub(crate) mod tests {
instance::ApubSite,
person::{tests::parse_lemmy_person, ApubPerson},
post::ApubPost,
tests::init_context,
},
protocol::tests::file_to_json_object,
};
@ -230,7 +232,7 @@ pub(crate) mod tests {
#[tokio::test]
#[serial]
pub(crate) async fn test_parse_lemmy_comment() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let url = Url::parse("https://enterprise.lemmy.ml/comment/38741")?;
let data = prepare_comment_test(&url, &context).await?;
@ -255,7 +257,7 @@ pub(crate) mod tests {
#[tokio::test]
#[serial]
async fn test_parse_pleroma_comment() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let url = Url::parse("https://enterprise.lemmy.ml/comment/38741")?;
let data = prepare_comment_test(&url, &context).await?;

52
crates/apub/src/objects/community.rs
View file

@ -2,7 +2,7 @@ use crate::{
activities::GetActorType,
check_apub_id_valid,
local_site_data_cached,
objects::instance::fetch_instance_actor_for_object,
objects::{instance::fetch_instance_actor_for_object, read_from_string_or_source_opt},
protocol::{
objects::{group::Group, Endpoints, LanguageTag},
ImageObject,
@ -17,15 +17,24 @@ use activitypub_federation::{
use chrono::{DateTime, Utc};
use lemmy_api_common::{
context::LemmyContext,
utils::{generate_featured_url, generate_moderators_url, generate_outbox_url},
utils::{
generate_featured_url,
generate_moderators_url,
generate_outbox_url,
local_site_opt_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_apub,
},
};
use lemmy_db_schema::{
source::{
activity::ActorType,
actor_language::CommunityLanguage,
community::{Community, CommunityUpdateForm},
community::{Community, CommunityInsertForm, CommunityUpdateForm},
local_site::LocalSite,
},
traits::{ApubActor, Crud},
utils::naive_now,
};
use lemmy_db_views_actor::structs::CommunityFollowerView;
use lemmy_utils::{error::LemmyError, spawn_try_task, utils::markdown::markdown_to_html};
@ -130,7 +139,38 @@ impl Object for ApubCommunity {
) -> Result<ApubCommunity, LemmyError> {
let instance_id = fetch_instance_actor_for_object(&group.id, context).await?;
let form = Group::into_insert_form(group.clone(), instance_id);
let local_site = LocalSite::read(&mut context.pool()).await.ok();
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let description = read_from_string_or_source_opt(&group.summary, &None, &group.source);
let description = process_markdown_opt(&description, slur_regex, context).await?;
let icon = proxy_image_link_opt_apub(group.icon.map(|i| i.url), context).await?;
let banner = proxy_image_link_opt_apub(group.image.map(|i| i.url), context).await?;
let form = CommunityInsertForm {
name: group.preferred_username.clone(),
title: group.name.unwrap_or(group.preferred_username.clone()),
description,
removed: None,
published: group.published,
updated: group.updated,
deleted: Some(false),
nsfw: Some(group.sensitive.unwrap_or(false)),
actor_id: Some(group.id.into()),
local: Some(false),
private_key: None,
hidden: None,
public_key: group.public_key.public_key_pem,
last_refreshed_at: Some(naive_now()),
icon,
banner,
followers_url: Some(group.followers.clone().into()),
inbox_url: Some(group.inbox.into()),
shared_inbox_url: group.endpoints.map(|e| e.shared_inbox.into()),
moderators_url: group.attributed_to.clone().map(Into::into),
posting_restricted_to_mods: group.posting_restricted_to_mods,
instance_id,
featured_url: group.featured.map(Into::into),
};
let languages =
LanguageTag::to_language_id_multiple(group.language, &mut context.pool()).await?;
@ -212,7 +252,7 @@ impl ApubCommunity {
pub(crate) mod tests {
use super::*;
use crate::{
objects::{instance::tests::parse_lemmy_instance, tests::init_context},
objects::instance::tests::parse_lemmy_instance,
protocol::tests::file_to_json_object,
};
use activitypub_federation::fetch::collection_id::CollectionId;
@ -241,7 +281,7 @@ pub(crate) mod tests {
#[tokio::test]
#[serial]
async fn test_parse_lemmy_community() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let site = parse_lemmy_instance(&context).await?;
let community = parse_lemmy_community(&context).await?;

30
crates/apub/src/objects/instance.rs
View file

@ -17,13 +17,17 @@ use activitypub_federation::{
traits::{Actor, Object},
};
use chrono::{DateTime, Utc};
use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex};
use lemmy_api_common::{
context::LemmyContext,
utils::{local_site_opt_to_slur_regex, process_markdown_opt, proxy_image_link_opt_apub},
};
use lemmy_db_schema::{
newtypes::InstanceId,
source::{
activity::ActorType,
actor_language::SiteLanguage,
instance::Instance as DbInstance,
local_site::LocalSite,
site::{Site, SiteInsertForm},
},
traits::Crud,
@ -126,18 +130,23 @@ impl Object for ApubSite {
}
#[tracing::instrument(skip_all)]
async fn from_json(apub: Self::Kind, data: &Data<Self::DataType>) -> Result<Self, LemmyError> {
async fn from_json(apub: Self::Kind, context: &Data<Self::DataType>) -> Result<Self, LemmyError> {
let domain = apub.id.inner().domain().expect("group id has domain");
let instance = DbInstance::read_or_create(&mut data.pool(), domain.to_string()).await?;
let instance = DbInstance::read_or_create(&mut context.pool(), domain.to_string()).await?;
let local_site = LocalSite::read(&mut context.pool()).await.ok();
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let sidebar = read_from_string_or_source_opt(&apub.content, &None, &apub.source);
let sidebar = process_markdown_opt(&sidebar, slur_regex, context).await?;
let icon = proxy_image_link_opt_apub(apub.icon.map(|i| i.url), context).await?;
let banner = proxy_image_link_opt_apub(apub.image.map(|i| i.url), context).await?;
let site_form = SiteInsertForm {
name: apub.name.clone(),
sidebar,
updated: apub.updated,
icon: apub.icon.clone().map(|i| i.url.into()),
banner: apub.image.clone().map(|i| i.url.into()),
icon,
banner,
description: apub.summary,
actor_id: Some(apub.id.clone().into()),
last_refreshed_at: Some(naive_now()),
@ -146,10 +155,11 @@ impl Object for ApubSite {
private_key: None,
instance_id: instance.id,
};
let languages = LanguageTag::to_language_id_multiple(apub.language, &mut data.pool()).await?;
let languages =
LanguageTag::to_language_id_multiple(apub.language, &mut context.pool()).await?;
let site = Site::create(&mut data.pool(), &site_form).await?;
SiteLanguage::update(&mut data.pool(), languages, &site).await?;
let site = Site::create(&mut context.pool(), &site_form).await?;
SiteLanguage::update(&mut context.pool(), languages, &site).await?;
Ok(site.into())
}
}
@ -205,7 +215,7 @@ pub(in crate::objects) async fn fetch_instance_actor_for_object<T: Into<Url> + C
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::{objects::tests::init_context, protocol::tests::file_to_json_object};
use crate::protocol::tests::file_to_json_object;
use lemmy_db_schema::traits::Crud;
use lemmy_utils::error::LemmyResult;
use pretty_assertions::assert_eq;
@ -223,7 +233,7 @@ pub(crate) mod tests {
#[tokio::test]
#[serial]
async fn test_parse_lemmy_instance() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let site = parse_lemmy_instance(&context).await?;
assert_eq!(site.name, "Enterprise");

51
crates/apub/src/objects/mod.rs
View file

@ -51,54 +51,3 @@ pub(crate) fn verify_is_remote_object(id: &Url, settings: &Settings) -> Result<(
Ok(())
}
}
#[cfg(test)]
pub(crate) mod tests {
use activitypub_federation::config::{Data, FederationConfig};
use anyhow::anyhow;
use lemmy_api_common::{context::LemmyContext, request::client_builder};
use lemmy_db_schema::{source::secret::Secret, utils::build_db_pool_for_tests};
use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell, settings::SETTINGS};
use reqwest::{Request, Response};
use reqwest_middleware::{ClientBuilder, Middleware, Next};
use task_local_extensions::Extensions;
struct BlockedMiddleware;
/// A reqwest middleware which blocks all requests
#[async_trait::async_trait]
impl Middleware for BlockedMiddleware {
async fn handle(
&self,
_req: Request,
_extensions: &mut Extensions,
_next: Next<'_>,
) -> reqwest_middleware::Result<Response> {
Err(anyhow!("Network requests not allowed").into())
}
}
// TODO: would be nice if we didnt have to use a full context for tests.
pub(crate) async fn init_context() -> LemmyResult<Data<LemmyContext>> {
// call this to run migrations
let pool = build_db_pool_for_tests().await;
let client = client_builder(&SETTINGS).build()?;
let client = ClientBuilder::new(client).with(BlockedMiddleware).build();
let secret = Secret {
id: 0,
jwt_secret: String::new(),
};
let rate_limit_cell = RateLimitCell::with_test_config();
let context = LemmyContext::create(pool, client, secret, rate_limit_cell.clone());
let config = FederationConfig::builder()
.domain("example.com")
.app_data(context)
.build()
.await?;
Ok(config.to_request_data())
}
}

26
crates/apub/src/objects/person.rs
View file

@ -20,11 +20,17 @@ use activitypub_federation::{
use chrono::{DateTime, Utc};
use lemmy_api_common::{
context::LemmyContext,
utils::{generate_outbox_url, local_site_opt_to_slur_regex},
utils::{
generate_outbox_url,
local_site_opt_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_apub,
},
};
use lemmy_db_schema::{
source::{
activity::ActorType,
local_site::LocalSite,
person::{Person as DbPerson, PersonInsertForm, PersonUpdateForm},
},
traits::{ApubActor, Crud},
@ -144,7 +150,12 @@ impl Object for ApubPerson {
) -> Result<ApubPerson, LemmyError> {
let instance_id = fetch_instance_actor_for_object(&person.id, context).await?;
let local_site = LocalSite::read(&mut context.pool()).await.ok();
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let bio = read_from_string_or_source_opt(&person.summary, &None, &person.source);
let bio = process_markdown_opt(&bio, slur_regex, context).await?;
let avatar = proxy_image_link_opt_apub(person.icon.map(|i| i.url), context).await?;
let banner = proxy_image_link_opt_apub(person.image.map(|i| i.url), context).await?;
// Some Mastodon users have `name: ""` (empty string), need to convert that to `None`
// https://github.com/mastodon/mastodon/issues/25233
@ -156,8 +167,8 @@ impl Object for ApubPerson {
banned: None,
ban_expires: None,
deleted: Some(false),
avatar: person.icon.map(|i| i.url.into()),
banner: person.image.map(|i| i.url.into()),
avatar,
banner,
published: person.published.map(Into::into),
updated: person.updated.map(Into::into),
actor_id: Some(person.id.into()),
@ -210,10 +221,7 @@ impl GetActorType for ApubPerson {
pub(crate) mod tests {
use super::*;
use crate::{
objects::{
instance::{tests::parse_lemmy_instance, ApubSite},
tests::init_context,
},
objects::instance::{tests::parse_lemmy_instance, ApubSite},
protocol::{objects::instance::Instance, tests::file_to_json_object},
};
use activitypub_federation::fetch::object_id::ObjectId;
@ -237,7 +245,7 @@ pub(crate) mod tests {
#[tokio::test]
#[serial]
async fn test_parse_lemmy_person() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let (person, site) = parse_lemmy_person(&context).await?;
assert_eq!(person.display_name, Some("Jean-Luc Picard".to_string()));
@ -251,7 +259,7 @@ pub(crate) mod tests {
#[tokio::test]
#[serial]
async fn test_parse_pleroma_person() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
// create and parse a fake pleroma instance actor, to avoid network request during test
let mut json: Instance = file_to_json_object("assets/lemmy/objects/instance.json")?;

72
crates/apub/src/objects/post.rs
View file

@ -24,8 +24,14 @@ use chrono::{DateTime, Utc};
use html2text::{from_read_with_decorator, render::text_renderer::TrivialDecorator};
use lemmy_api_common::{
context::LemmyContext,
request::fetch_site_data,
utils::{is_mod_or_admin, local_site_opt_to_sensitive, local_site_opt_to_slur_regex},
request::fetch_link_metadata_opt,
utils::{
is_mod_or_admin,
local_site_opt_to_sensitive,
local_site_opt_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_apub,
},
};
use lemmy_db_schema::{
self,
@ -40,11 +46,7 @@ use lemmy_db_schema::{
};
use lemmy_utils::{
error::LemmyError,
utils::{
markdown::markdown_to_html,
slurs::{check_slurs_opt, remove_slurs},
validation::check_url_scheme,
},
utils::{markdown::markdown_to_html, slurs::check_slurs_opt, validation::check_url_scheme},
};
use std::ops::Deref;
use stringreader::StringReader;
@ -111,6 +113,13 @@ impl Object for ApubPost {
let community = Community::read(&mut context.pool(), community_id).await?;
let language = LanguageTag::new_single(self.language_id, &mut context.pool()).await?;
let attachment = self
.url
.clone()
.map(|url| Attachment::new(url.into(), self.url_content_type.clone()))
.into_iter()
.collect();
let page = Page {
kind: PageType::Page,
id: self.ap_id.clone().into(),
@ -121,7 +130,7 @@ impl Object for ApubPost {
content: self.body.as_ref().map(|b| markdown_to_html(b)),
media_type: Some(MediaTypeMarkdownOrHtml::Html),
source: self.body.clone().map(Source::new),
attachment: self.url.clone().map(Attachment::new).into_iter().collect(),
attachment,
image: self.thumbnail_url.clone().map(ImageObject::new),
comments_enabled: Some(!self.locked),
sensitive: Some(self.nsfw),
@ -210,33 +219,22 @@ impl Object for ApubPost {
let local_site = LocalSite::read(&mut context.pool()).await.ok();
let allow_sensitive = local_site_opt_to_sensitive(&local_site);
let page_is_sensitive = page.sensitive.unwrap_or(false);
let include_image = allow_sensitive || !page_is_sensitive;
let allow_generate_thumbnail = allow_sensitive || !page_is_sensitive;
let mut thumbnail_url = page.image.map(|i| i.url);
let do_generate_thumbnail = thumbnail_url.is_none() && allow_generate_thumbnail;
// Only fetch metadata if the post has a url and was not seen previously. We dont want to
// waste resources by fetching metadata for the same post multiple times.
// Additionally, only fetch image if content is not sensitive or is allowed on local site.
let (metadata_res, thumbnail) = match &url {
Some(url) if old_post.is_err() => {
fetch_site_data(
context.client(),
context.settings(),
Some(url),
include_image,
)
.await
}
_ => (None, None),
};
// If no image was included with metadata, use post image instead when available.
let thumbnail_url = thumbnail.or_else(|| page.image.map(|i| i.url.into()));
// Generate local thumbnail only if no thumbnail was federated and 'sensitive' attributes allow it.
let metadata = fetch_link_metadata_opt(url.as_ref(), do_generate_thumbnail, context).await;
if let Some(thumbnail_url_) = metadata.thumbnail {
thumbnail_url = Some(thumbnail_url_.into());
}
let url = proxy_image_link_opt_apub(url, context).await?;
let thumbnail_url = proxy_image_link_opt_apub(thumbnail_url, context).await?;
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (u.title, u.description, u.embed_video_url))
.unwrap_or_default();
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let body = read_from_string_or_source_opt(&page.content, &page.media_type, &page.source)
.map(|s| remove_slurs(&s, slur_regex));
let body = read_from_string_or_source_opt(&page.content, &page.media_type, &page.source);
let body = process_markdown_opt(&body, slur_regex, context).await?;
let language_id =
LanguageTag::to_language_id_single(page.language, &mut context.pool()).await?;
@ -252,15 +250,16 @@ impl Object for ApubPost {
updated: page.updated.map(Into::into),
deleted: Some(false),
nsfw: page.sensitive,
embed_title,
embed_description,
embed_video_url,
embed_title: metadata.opengraph_data.title,
embed_description: metadata.opengraph_data.description,
embed_video_url: metadata.opengraph_data.embed_video_url,
thumbnail_url,
ap_id: Some(page.id.clone().into()),
local: Some(false),
language_id,
featured_community: None,
featured_local: None,
url_content_type: metadata.content_type,
}
} else {
// if is mod action, only update locked/stickied fields, nothing else
@ -299,7 +298,6 @@ mod tests {
instance::ApubSite,
person::{tests::parse_lemmy_person, ApubPerson},
post::ApubPost,
tests::init_context,
},
protocol::tests::file_to_json_object,
};
@ -311,7 +309,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn test_parse_lemmy_post() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let (person, site) = parse_lemmy_person(&context).await?;
let community = parse_lemmy_community(&context).await?;
@ -335,7 +333,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn test_convert_mastodon_post_title() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let (person, site) = parse_lemmy_person(&context).await?;
let community = parse_lemmy_community(&context).await?;

14
crates/apub/src/objects/private_message.rs
View file

@ -12,9 +12,13 @@ use activitypub_federation::{
traits::Object,
};
use chrono::{DateTime, Utc};
use lemmy_api_common::{context::LemmyContext, utils::check_person_block};
use lemmy_api_common::{
context::LemmyContext,
utils::{check_person_block, local_site_opt_to_slur_regex, process_markdown},
};
use lemmy_db_schema::{
source::{
local_site::LocalSite,
person::Person,
private_message::{PrivateMessage, PrivateMessageInsertForm},
},
@ -121,7 +125,10 @@ impl Object for ApubPrivateMessage {
let recipient = note.to[0].dereference(context).await?;
check_person_block(creator.id, recipient.id, &mut context.pool()).await?;
let local_site = LocalSite::read(&mut context.pool()).await.ok();
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let content = read_from_string_or_source(&note.content, &None, &note.source);
let content = process_markdown(&content, slur_regex, context).await?;
let form = PrivateMessageInsertForm {
creator_id: creator.id,
@ -146,7 +153,6 @@ mod tests {
objects::{
instance::{tests::parse_lemmy_instance, ApubSite},
person::ApubPerson,
tests::init_context,
},
protocol::tests::file_to_json_object,
};
@ -185,7 +191,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn test_parse_lemmy_pm() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let url = Url::parse("https://enterprise.lemmy.ml/private_message/1621")?;
let data = prepare_comment_test(&url, &context).await?;
let json: ChatMessage = file_to_json_object("assets/lemmy/objects/chat_message.json")?;
@ -208,7 +214,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn test_parse_pleroma_pm() -> LemmyResult<()> {
let context = init_context().await?;
let context = LemmyContext::init_test_context().await;
let url = Url::parse("https://enterprise.lemmy.ml/private_message/1621")?;
let data = prepare_comment_test(&url, &context).await?;
let pleroma_url = Url::parse("https://queer.hacktivis.me/objects/2")?;

65
crates/apub/src/protocol/objects/group.rs
View file

@ -25,11 +25,6 @@ use activitypub_federation::{
};
use chrono::{DateTime, Utc};
use lemmy_api_common::{context::LemmyContext, utils::local_site_opt_to_slur_regex};
use lemmy_db_schema::{
newtypes::InstanceId,
source::community::{CommunityInsertForm, CommunityUpdateForm},
utils::naive_now,
};
use lemmy_utils::{
error::LemmyError,
utils::slurs::{check_slurs, check_slurs_opt},
@ -94,64 +89,4 @@ impl Group {
check_slurs_opt(&description, slur_regex)?;
Ok(())
}
pub(crate) fn into_insert_form(self, instance_id: InstanceId) -> CommunityInsertForm {
let description = read_from_string_or_source_opt(&self.summary, &None, &self.source);
CommunityInsertForm {
name: self.preferred_username.clone(),
title: self.name.unwrap_or(self.preferred_username.clone()),
description,
removed: None,
published: self.published,
updated: self.updated,
deleted: Some(false),
nsfw: Some(self.sensitive.unwrap_or(false)),
actor_id: Some(self.id.into()),
local: Some(false),
private_key: None,
hidden: None,
public_key: self.public_key.public_key_pem,
last_refreshed_at: Some(naive_now()),
icon: self.icon.map(|i| i.url.into()),
banner: self.image.map(|i| i.url.into()),
followers_url: Some(self.followers.into()),
inbox_url: Some(self.inbox.into()),
shared_inbox_url: self.endpoints.map(|e| e.shared_inbox.into()),
moderators_url: self.attributed_to.map(Into::into),
posting_restricted_to_mods: self.posting_restricted_to_mods,
instance_id,
featured_url: self.featured.map(Into::into),
}
}
pub(crate) fn into_update_form(self) -> CommunityUpdateForm {
CommunityUpdateForm {
title: Some(self.name.unwrap_or(self.preferred_username)),
description: Some(read_from_string_or_source_opt(
&self.summary,
&None,
&self.source,
)),
removed: None,
published: self.published.map(Into::into),
updated: Some(self.updated.map(Into::into)),
deleted: None,
nsfw: Some(self.sensitive.unwrap_or(false)),
actor_id: Some(self.id.into()),
local: None,
private_key: None,
hidden: None,
public_key: Some(self.public_key.public_key_pem),
last_refreshed_at: Some(naive_now()),
icon: Some(self.icon.map(|i| i.url.into())),
banner: Some(self.image.map(|i| i.url.into())),
followers_url: Some(self.followers.into()),
inbox_url: Some(self.inbox.into()),
shared_inbox_url: Some(self.endpoints.map(|e| e.shared_inbox.into())),
moderators_url: self.attributed_to.map(Into::into),
posting_restricted_to_mods: self.posting_restricted_to_mods,
featured_url: self.featured.map(Into::into),
}
}
}

34
crates/apub/src/protocol/objects/page.rs
View file

@ -20,7 +20,6 @@ use activitypub_federation::{
use chrono::{DateTime, Utc};
use itertools::Itertools;
use lemmy_api_common::context::LemmyContext;
use lemmy_db_schema::newtypes::DbUrl;
use lemmy_utils::error::{LemmyError, LemmyErrorType};
use serde::{de::Error, Deserialize, Deserializer, Serialize};
use serde_with::skip_serializing_none;
@ -72,24 +71,25 @@ pub struct Page {
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct Link {
pub(crate) href: Url,
pub(crate) r#type: LinkType,
href: Url,
media_type: Option<String>,
r#type: LinkType,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct Image {
#[serde(rename = "type")]
pub(crate) kind: ImageType,
pub(crate) url: Url,
kind: ImageType,
url: Url,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct Document {
#[serde(rename = "type")]
pub(crate) kind: DocumentType,
pub(crate) url: Url,
kind: DocumentType,
url: Url,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
@ -167,11 +167,21 @@ impl Page {
}
impl Attachment {
pub(crate) fn new(url: DbUrl) -> Attachment {
Attachment::Link(Link {
href: url.into(),
r#type: Default::default(),
})
/// Creates new attachment for a given link and mime type.
pub(crate) fn new(url: Url, media_type: Option<String>) -> Attachment {
let is_image = media_type.clone().unwrap_or_default().starts_with("image");
if is_image {
Attachment::Image(Image {
kind: Default::default(),
url,
})
} else {
Attachment::Link(Link {
href: url,
media_type,
r#type: Default::default(),
})
}
}
}

35
crates/db_schema/src/impls/image_upload.rs
View file

@ -1,35 +0,0 @@
use crate::{
newtypes::LocalUserId,
schema::image_upload::dsl::{image_upload, local_user_id},
source::image_upload::{ImageUpload, ImageUploadForm},
utils::{get_conn, DbPool},
};
use diesel::{insert_into, result::Error, ExpressionMethods, QueryDsl, Table};
use diesel_async::RunQueryDsl;
impl ImageUpload {
pub async fn create(pool: &mut DbPool<'_>, form: &ImageUploadForm) -> Result<Self, Error> {
let conn = &mut get_conn(pool).await?;
insert_into(image_upload)
.values(form)
.get_result::<Self>(conn)
.await
}
pub async fn get_all_by_local_user_id(
pool: &mut DbPool<'_>,
user_id: &LocalUserId,
) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
image_upload
.filter(local_user_id.eq(user_id))
.select(image_upload::all_columns())
.load::<ImageUpload>(conn)
.await
}
pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?;
diesel::delete(image_upload.find(alias)).execute(conn).await
}
}

78
crates/db_schema/src/impls/images.rs Normal file
View file

@ -0,0 +1,78 @@
use crate::{
newtypes::{DbUrl, LocalUserId},
schema::{
local_image::dsl::{local_image, local_user_id, pictrs_alias},
remote_image::dsl::{link, remote_image},
},
source::images::{LocalImage, LocalImageForm, RemoteImage, RemoteImageForm},
utils::{get_conn, DbPool},
};
use diesel::{
dsl::exists,
insert_into,
result::Error,
select,
ExpressionMethods,
NotFound,
QueryDsl,
Table,
};
use diesel_async::RunQueryDsl;
use url::Url;
impl LocalImage {
pub async fn create(pool: &mut DbPool<'_>, form: &LocalImageForm) -> Result<Self, Error> {
let conn = &mut get_conn(pool).await?;
insert_into(local_image)
.values(form)
.get_result::<Self>(conn)
.await
}
pub async fn get_all_by_local_user_id(
pool: &mut DbPool<'_>,
user_id: &LocalUserId,
) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
local_image
.filter(local_user_id.eq(user_id))
.select(local_image::all_columns())
.load::<LocalImage>(conn)
.await
}
pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?;
diesel::delete(local_image.filter(pictrs_alias.eq(alias)))
.execute(conn)
.await
}
}
impl RemoteImage {
pub async fn create(pool: &mut DbPool<'_>, links: Vec<Url>) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?;
let forms = links
.into_iter()
.map(|url| RemoteImageForm { link: url.into() })
.collect::<Vec<_>>();
insert_into(remote_image)
.values(forms)
.on_conflict_do_nothing()
.execute(conn)
.await
}
pub async fn validate(pool: &mut DbPool<'_>, link_: DbUrl) -> Result<(), Error> {
let conn = &mut get_conn(pool).await?;
let exists = select(exists(remote_image.filter((link).eq(link_))))
.get_result::<bool>(conn)
.await?;
if exists {
Ok(())
} else {
Err(NotFound)
}
}
}

2
crates/db_schema/src/impls/mod.rs
View file

@ -11,7 +11,7 @@ pub mod email_verification;
pub mod federation_allowlist;
pub mod federation_blocklist;
pub mod federation_queue_state;
pub mod image_upload;
pub mod images;
pub mod instance;
pub mod instance_block;
pub mod language;

1
crates/db_schema/src/impls/post.rs
View file

@ -434,6 +434,7 @@ mod tests {
language_id: Default::default(),
featured_community: false,
featured_local: false,
url_content_type: None,
};
// Post Like

32
crates/db_schema/src/schema.rs
View file

@ -301,15 +301,6 @@ diesel::table! {
}
}
diesel::table! {
image_upload (pictrs_alias) {
local_user_id -> Int4,
pictrs_alias -> Text,
pictrs_delete_token -> Text,
published -> Timestamptz,
}
}
diesel::table! {
instance (id) {
id -> Int4,
@ -341,6 +332,15 @@ diesel::table! {
}
}
diesel::table! {
local_image (pictrs_alias) {
local_user_id -> Int4,
pictrs_alias -> Text,
pictrs_delete_token -> Text,
published -> Timestamptz,
}
}
diesel::table! {
use diesel::sql_types::*;
use super::sql_types::ListingTypeEnum;
@ -692,6 +692,7 @@ diesel::table! {
language_id -> Int4,
featured_community -> Bool,
featured_local -> Bool,
url_content_type -> Nullable<Text>,
}
}
@ -807,6 +808,14 @@ diesel::table! {
}
}
diesel::table! {
remote_image (id) {
id -> Int4,
link -> Text,
published -> Timestamptz,
}
}
diesel::table! {
secret (id) {
id -> Int4,
@ -922,9 +931,9 @@ diesel::joinable!(email_verification -> local_user (local_user_id));
diesel::joinable!(federation_allowlist -> instance (instance_id));
diesel::joinable!(federation_blocklist -> instance (instance_id));
diesel::joinable!(federation_queue_state -> instance (instance_id));
diesel::joinable!(image_upload -> local_user (local_user_id));
diesel::joinable!(instance_block -> instance (instance_id));
diesel::joinable!(instance_block -> person (person_id));
diesel::joinable!(local_image -> local_user (local_user_id));
diesel::joinable!(local_site -> site (site_id));
diesel::joinable!(local_site_rate_limit -> local_site (local_site_id));
diesel::joinable!(local_user -> person (person_id));
@ -1002,10 +1011,10 @@ diesel::allow_tables_to_appear_in_same_query!(
federation_allowlist,
federation_blocklist,
federation_queue_state,
image_upload,
instance,
instance_block,
language,
local_image,
local_site,
local_site_rate_limit,
local_user,
@ -1040,6 +1049,7 @@ diesel::allow_tables_to_appear_in_same_query!(
private_message_report,
received_activity,
registration_application,
remote_image,
secret,
sent_activity,
site,

50
crates/db_schema/src/source/images.rs Normal file
View file

@ -0,0 +1,50 @@
use crate::newtypes::{DbUrl, LocalUserId};
#[cfg(feature = "full")]
use crate::schema::{local_image, remote_image};
use chrono::{DateTime, Utc};
use serde_with::skip_serializing_none;
use std::fmt::Debug;
use typed_builder::TypedBuilder;
#[skip_serializing_none]
#[derive(PartialEq, Eq, Debug, Clone)]
#[cfg_attr(feature = "full", derive(Queryable, Associations))]
#[cfg_attr(feature = "full", diesel(table_name = local_image))]
#[cfg_attr(
feature = "full",
diesel(belongs_to(crate::source::local_user::LocalUser))
)]
pub struct LocalImage {
pub local_user_id: LocalUserId,
pub pictrs_alias: String,
pub pictrs_delete_token: String,
pub published: DateTime<Utc>,
}
#[derive(Debug, Clone, TypedBuilder)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = local_image))]
pub struct LocalImageForm {
pub local_user_id: LocalUserId,
pub pictrs_alias: String,
pub pictrs_delete_token: String,
}
/// Stores all images which are hosted on remote domains. When attempting to proxy an image, it
/// is checked against this table to avoid Lemmy being used as a general purpose proxy.
#[skip_serializing_none]
#[derive(PartialEq, Eq, Debug, Clone)]
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
pub struct RemoteImage {
pub id: i32,
pub link: DbUrl,
pub published: DateTime<Utc>,
}
#[derive(Debug, Clone, TypedBuilder)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
pub struct RemoteImageForm {
pub link: DbUrl,
}

2
crates/db_schema/src/source/mod.rs
View file

@ -16,7 +16,7 @@ pub mod email_verification;
pub mod federation_allowlist;
pub mod federation_blocklist;
pub mod federation_queue_state;
pub mod image_upload;
pub mod images;
pub mod instance;
pub mod instance_block;
pub mod language;

3
crates/db_schema/src/source/post.rs
View file

@ -55,6 +55,7 @@ pub struct Post {
pub featured_community: bool,
/// Whether the post is featured to its site.
pub featured_local: bool,
pub url_content_type: Option<String>,
}
#[derive(Debug, Clone, TypedBuilder)]
@ -85,6 +86,7 @@ pub struct PostInsertForm {
pub language_id: Option<LanguageId>,
pub featured_community: Option<bool>,
pub featured_local: Option<bool>,
pub url_content_type: Option<String>,
}
#[derive(Debug, Clone, Default)]
@ -109,6 +111,7 @@ pub struct PostUpdateForm {
pub language_id: Option<LanguageId>,
pub featured_community: Option<bool>,
pub featured_local: Option<bool>,
pub url_content_type: Option<String>,
}
#[derive(PartialEq, Eq, Debug)]

1
crates/db_views/src/comment_view.rs
View file

@ -1015,6 +1015,7 @@ mod tests {
language_id: Default::default(),
featured_community: false,
featured_local: false,
url_content_type: None,
},
community: Community {
id: data.inserted_community.id,

1
crates/db_views/src/post_view.rs
View file

@ -1468,6 +1468,7 @@ mod tests {
language_id: LanguageId(47),
featured_community: false,
featured_local: false,
url_content_type: None,
},
my_vote: None,
unread_comments: 0,

1
crates/routes/Cargo.toml
View file

@ -33,4 +33,5 @@ url = { workspace = true }
once_cell = { workspace = true }
tracing = { workspace = true }
tokio = { workspace = true }
urlencoding = { workspace = true }
rss = "2.0.7"

44
crates/routes/src/images.rs
View file

@ -6,6 +6,7 @@ use actix_web::{
StatusCode,
},
web,
web::Query,
Error,
HttpRequest,
HttpResponse,
@ -13,15 +14,17 @@ use actix_web::{
use futures::stream::{Stream, StreamExt};
use lemmy_api_common::context::LemmyContext;
use lemmy_db_schema::source::{
image_upload::{ImageUpload, ImageUploadForm},
images::{LocalImage, LocalImageForm, RemoteImage},
local_site::LocalSite,
};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::{rate_limit::RateLimitCell, REQWEST_TIMEOUT};
use lemmy_utils::{error::LemmyResult, rate_limit::RateLimitCell, REQWEST_TIMEOUT};
use reqwest::Body;
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
use serde::{Deserialize, Serialize};
use std::time::Duration;
use url::Url;
use urlencoding::decode;
pub fn config(
cfg: &mut web::ServiceConfig,
@ -87,13 +90,14 @@ async fn upload(
body: web::Payload,
// require login
local_user_view: LocalUserView,
client: web::Data<ClientWithMiddleware>,
context: web::Data<LemmyContext>,
) -> Result<HttpResponse, Error> {
// TODO: check rate limit here
let pictrs_config = context.settings().pictrs_config()?;
let image_url = format!("{}image", pictrs_config.url);
let mut client_req = adapt_request(&req, context.client(), image_url);
let mut client_req = adapt_request(&req, &client, image_url);
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string())
@ -109,12 +113,12 @@ async fn upload(
let images = res.json::<Images>().await.map_err(error::ErrorBadRequest)?;
if let Some(images) = &images.files {
for uploaded_image in images {
let form = ImageUploadForm {
let form = LocalImageForm {
local_user_id: local_user_view.local_user.id,
pictrs_alias: uploaded_image.file.to_string(),
pictrs_delete_token: uploaded_image.delete_token.to_string(),
};
ImageUpload::create(&mut context.pool(), &form)
LocalImage::create(&mut context.pool(), &form)
.await
.map_err(error::ErrorBadRequest)?;
}
@ -158,15 +162,15 @@ async fn full_res(
url
};
image(url, req, client).await
image(url, req, &client).await
}
async fn image(
url: String,
req: HttpRequest,
client: web::Data<ClientWithMiddleware>,
client: &ClientWithMiddleware,
) -> Result<HttpResponse, Error> {
let mut client_req = adapt_request(&req, &client, url);
let mut client_req = adapt_request(&req, client, url);
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string());
@ -212,13 +216,35 @@ async fn delete(
let res = client_req.send().await.map_err(error::ErrorBadRequest)?;
ImageUpload::delete_by_alias(&mut context.pool(), &file)
LocalImage::delete_by_alias(&mut context.pool(), &file)
.await
.map_err(error::ErrorBadRequest)?;
Ok(HttpResponse::build(res.status()).body(BodyStream::new(res.bytes_stream())))
}
#[derive(Deserialize)]
pub struct ImageProxyParams {
url: String,
}
pub async fn image_proxy(
Query(params): Query<ImageProxyParams>,
context: web::Data<LemmyContext>,
) -> LemmyResult<HttpResponse> {
let url = Url::parse(&decode(&params.url)?)?;
// Check that url corresponds to a federated image so that this can't be abused as a proxy
// for arbitrary purposes.
RemoteImage::validate(&mut context.pool(), url.clone().into()).await?;
let pictrs_config = context.settings().pictrs_config()?;
let url = format!("{}image/original?proxy={}", pictrs_config.url, &params.url);
let image_response = context.client().get(url).send().await?;
Ok(HttpResponse::Ok().streaming(image_response.bytes_stream()))
}
fn make_send<S>(mut stream: S) -> impl Stream<Item = S::Item> + Send + Unpin + 'static
where
S: Stream + Unpin + 'static,

2
crates/utils/Cargo.toml
View file

@ -39,8 +39,8 @@ http = { workspace = true }
doku = { workspace = true, features = ["url-2"] }
uuid = { workspace = true, features = ["serde", "v4"] }
rosetta-i18n = { workspace = true }
percent-encoding = { workspace = true }
tokio = { workspace = true }
urlencoding = { workspace = true }
openssl = "0.10.63"
html2text = "0.6.0"
deser-hjson = "2.2.4"

23
crates/utils/src/settings/mod.rs
View file

@ -6,12 +6,13 @@ use crate::{
use anyhow::{anyhow, Context};
use deser_hjson::from_str;
use once_cell::sync::Lazy;
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use regex::Regex;
use std::{env, fs, io::Error};
use urlencoding::encode;
pub mod structs;
use crate::settings::structs::PictrsImageMode;
use structs::DatabaseConnection;
static DEFAULT_CONFIG_FILE: &str = "config/config.hjson";
@ -53,11 +54,11 @@ impl Settings {
DatabaseConnection::Parts(parts) => {
format!(
"postgres://{}:{}@{}:{}/{}",
utf8_percent_encode(&parts.user, NON_ALPHANUMERIC),
utf8_percent_encode(&parts.password, NON_ALPHANUMERIC),
encode(&parts.user),
encode(&parts.password),
parts.host,
parts.port,
utf8_percent_encode(&parts.database, NON_ALPHANUMERIC),
encode(&parts.database),
)
}
}
@ -112,3 +113,17 @@ impl Settings {
.ok_or_else(|| anyhow!("images_disabled").into())
}
}
impl PictrsConfig {
pub fn image_mode(&self) -> PictrsImageMode {
if let Some(cache_external_link_previews) = self.cache_external_link_previews {
if cache_external_link_previews {
PictrsImageMode::StoreLinkPreviews
} else {
PictrsImageMode::None
}
} else {
self.image_mode.clone()
}
}
}

42
crates/utils/src/settings/structs.rs
View file

@ -12,7 +12,6 @@ pub struct Settings {
/// settings related to the postgresql database
#[default(Default::default())]
pub database: DatabaseConfig,
/// Settings related to activitypub federation
/// Pictrs image server configuration.
#[default(Some(Default::default()))]
pub(crate) pictrs: Option<PictrsConfig>,
@ -79,22 +78,43 @@ pub struct PictrsConfig {
#[default(None)]
pub api_key: Option<String>,
/// By default the thumbnails for external links are stored in pict-rs. This ensures that they
/// can be reliably retrieved and can be resized using pict-rs APIs. However it also increases
/// storage usage. In case this is disabled, the Opengraph image is directly returned as
/// thumbnail.
/// Backwards compatibility with 0.18.1. False is equivalent to `image_mode: None`, true is
/// equivalent to `image_mode: StoreLinkPreviews`.
///
/// In some countries it is forbidden to copy preview images from newspaper articles and only
/// hotlinking is allowed. If that is the case for your instance, make sure that this setting is
/// disabled.
#[default(true)]
pub cache_external_link_previews: bool,
/// To be removed in 0.20
pub(super) cache_external_link_previews: Option<bool>,
/// Timeout for uploading images to pictrs (in seconds)
/// Specifies how to handle remote images, so that users don't have to connect directly to remote servers.
#[default(PictrsImageMode::StoreLinkPreviews)]
pub(super) image_mode: PictrsImageMode,
/// Timeout for uploading images to pictrs (in seconds)
#[default(30)]
pub upload_timeout: u64,
}
#[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document, PartialEq)]
#[serde(deny_unknown_fields)]
pub enum PictrsImageMode {
/// Leave images unchanged, don't generate any local thumbnails for post urls. Instead the the
/// Opengraph image is directly returned as thumbnail
None,
/// Generate thumbnails for external post urls and store them persistently in pict-rs. This
/// ensures that they can be reliably retrieved and can be resized using pict-rs APIs. However
/// it also increases storage usage.
///
/// This is the default behaviour, and also matches Lemmy 0.18.
#[default]
StoreLinkPreviews,
/// If enabled, all images from remote domains are rewritten to pass through `/api/v3/image_proxy`,
/// including embedded images in markdown. Images are stored temporarily in pict-rs for caching.
/// This improves privacy as users don't expose their IP to untrusted servers, and decreases load
/// on other servers. However it increases bandwidth use for the local server.
///
/// Requires pict-rs 0.5
ProxyAllImages,
}
#[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)]
#[serde(default)]
pub struct DatabaseConfig {

113
crates/utils/src/utils/markdown.rs
View file

@ -1,113 +0,0 @@
use markdown_it::MarkdownIt;
use once_cell::sync::Lazy;
mod spoiler_rule;
static MARKDOWN_PARSER: Lazy<MarkdownIt> = Lazy::new(|| {
let mut parser = MarkdownIt::new();
markdown_it::plugins::cmark::add(&mut parser);
markdown_it::plugins::extra::add(&mut parser);
spoiler_rule::add(&mut parser);
parser
});
/// Replace special HTML characters in API parameters to prevent XSS attacks.
///
/// Taken from https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.md#output-encoding-for-html-contexts
///
/// `>` is left in place because it is interpreted as markdown quote.
pub fn sanitize_html(text: &str) -> String {
text
.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('\"', "&quot;")
.replace('\'', "&#x27;")
}
/// Converts text from markdown to HTML, while escaping special characters.
pub fn markdown_to_html(text: &str) -> String {
MARKDOWN_PARSER.parse(text).xrender()
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
#![allow(clippy::indexing_slicing)]
use super::*;
use pretty_assertions::assert_eq;
#[test]
fn test_basic_markdown() {
let tests: Vec<_> = vec![
(
"headings",
"# h1\n## h2\n### h3\n#### h4\n##### h5\n###### h6",
"<h1>h1</h1>\n<h2>h2</h2>\n<h3>h3</h3>\n<h4>h4</h4>\n<h5>h5</h5>\n<h6>h6</h6>\n"
),
(
"line breaks",
"First\rSecond",
"<p>First\nSecond</p>\n"),
(
"emphasis",
"__bold__ **bold** *italic* ***bold+italic***",
"<p><strong>bold</strong> <strong>bold</strong> <em>italic</em> <em><strong>bold+italic</strong></em></p>\n"
),
(
"blockquotes",
"> #### Hello\n > \n > - Hola\n > - 안영 \n>> Goodbye\n",
"<blockquote>\n<h4>Hello</h4>\n<ul>\n<li>Hola</li>\n<li>안영</li>\n</ul>\n<blockquote>\n<p>Goodbye</p>\n</blockquote>\n</blockquote>\n"
),
(
"lists (ordered, unordered)",
"1. pen\n2. apple\n3. apple pen\n- pen\n- pineapple\n- pineapple pen",
"<ol>\n<li>pen</li>\n<li>apple</li>\n<li>apple pen</li>\n</ol>\n<ul>\n<li>pen</li>\n<li>pineapple</li>\n<li>pineapple pen</li>\n</ul>\n"
),
(
"code and code blocks",
"this is my amazing `code snippet` and my amazing ```code block```",
"<p>this is my amazing <code>code snippet</code> and my amazing <code>code block</code></p>\n"
),
(
"links",
"[Lemmy](https://join-lemmy.org/ \"Join Lemmy!\")",
"<p><a href=\"https://join-lemmy.org/\" title=\"Join Lemmy!\">Lemmy</a></p>\n"
),
(
"images",
"![My linked image](https://image.com \"image alt text\")",
"<p><img src=\"https://image.com\" alt=\"My linked image\" title=\"image alt text\" /></p>\n"
),
// Ensure any custom plugins are added to 'MARKDOWN_PARSER' implementation.
(
"basic spoiler",
"::: spoiler click to see more\nhow spicy!\n:::\n",
"<details><summary>click to see more</summary><p>how spicy!\n</p></details>\n"
),
(
"escape html special chars",
"<script>alert('xss');</script> hello &\"",
"<p>&lt;script&gt;alert(‘xss’);&lt;/script&gt; hello &amp;&quot;</p>\n"
)
];
tests.iter().for_each(|&(msg, input, expected)| {
let result = markdown_to_html(input);
assert_eq!(
result, expected,
"Testing {}, with original input '{}'",
msg, input
);
});
}
#[test]
fn test_sanitize_html() {
let sanitized = sanitize_html("<script>alert('xss');</script> hello &\"'");
let expected = "&lt;script>alert(&#x27;xss&#x27;);&lt;/script> hello &amp;&quot;&#x27;";
assert_eq!(expected, sanitized)
}
}

38
crates/utils/src/utils/markdown/link_rule.rs Normal file
View file

@ -0,0 +1,38 @@
use markdown_it::{generics::inline::full_link, MarkdownIt, Node, NodeValue, Renderer};
/// Renders markdown links. Copied directly from markdown-it source, unlike original code it also
/// sets `rel=nofollow` attribute.
///
/// TODO: We can set nofollow only if post was not made by mod/admin, but then we have to construct
/// new parser for every invocation which might have performance implications.
/// https://github.com/markdown-it-rust/markdown-it/blob/master/src/plugins/cmark/inline/link.rs
#[derive(Debug)]
pub struct Link {
pub url: String,
pub title: Option<String>,
}
impl NodeValue for Link {
fn render(&self, node: &Node, fmt: &mut dyn Renderer) {
let mut attrs = node.attrs.clone();
attrs.push(("href", self.url.clone()));
attrs.push(("rel", "nofollow".to_string()));
if let Some(title) = &self.title {
attrs.push(("title", title.clone()));
}
fmt.open("a", &attrs);
fmt.contents(&node.children);
fmt.close("a");
}
}
pub fn add(md: &mut MarkdownIt) {
full_link::add::<false>(md, |href, title| {
Node::new(Link {
url: href.unwrap_or_default(),
title,
})
});
}

246
crates/utils/src/utils/markdown/mod.rs Normal file
View file

@ -0,0 +1,246 @@
use crate::settings::SETTINGS;
use markdown_it::{plugins::cmark::inline::image::Image, MarkdownIt};
use once_cell::sync::Lazy;
use url::Url;
use urlencoding::encode;
mod link_rule;
mod spoiler_rule;
static MARKDOWN_PARSER: Lazy<MarkdownIt> = Lazy::new(|| {
let mut parser = MarkdownIt::new();
markdown_it::plugins::cmark::add(&mut parser);
markdown_it::plugins::extra::add(&mut parser);
spoiler_rule::add(&mut parser);
link_rule::add(&mut parser);
parser
});
/// Replace special HTML characters in API parameters to prevent XSS attacks.
///
/// Taken from https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.md#output-encoding-for-html-contexts
///
/// `>` is left in place because it is interpreted as markdown quote.
pub fn sanitize_html(text: &str) -> String {
text
.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('\"', "&quot;")
.replace('\'', "&#x27;")
}
pub fn markdown_to_html(text: &str) -> String {
MARKDOWN_PARSER.parse(text).xrender()
}
/// Rewrites all links to remote domains in markdown, so they go through `/api/v3/image_proxy`.
pub fn markdown_rewrite_image_links(mut src: String) -> (String, Vec<Url>) {
let ast = MARKDOWN_PARSER.parse(&src);
let mut links_offsets = vec![];
// Walk the syntax tree to find positions of image links
ast.walk(|node, _depth| {
if let Some(image) = node.cast::<Image>() {
// srcmap is always present for image
// https://github.com/markdown-it-rust/markdown-it/issues/36#issuecomment-1777844387
let node_offsets = node.srcmap.expect("srcmap is none").get_byte_offsets();
// necessary for custom emojis which look like `![name](url "title")`
let start_offset = node_offsets.1
- image.url.len()
- 1
- image
.title
.as_ref()
.map(|t| t.len() + 3)
.unwrap_or_default();
let end_offset = node_offsets.1 - 1;
links_offsets.push((start_offset, end_offset));
}
});
let mut links = vec![];
// Go through the collected links in reverse order
while let Some((start, end)) = links_offsets.pop() {
let content = src.get(start..end).unwrap_or_default();
// necessary for custom emojis which look like `![name](url "title")`
let (url, extra) = if content.contains(' ') {
let split = content.split_once(' ').expect("split is valid");
(split.0, Some(split.1))
} else {
(content, None)
};
match Url::parse(url) {
Ok(parsed) => {
links.push(parsed.clone());
// If link points to remote domain, replace with proxied link
if parsed.domain() != Some(&SETTINGS.hostname) {
let mut proxied = format!(
"{}/api/v3/image_proxy?url={}",
SETTINGS.get_protocol_and_hostname(),
encode(url),
);
// restore custom emoji format
if let Some(extra) = extra {
proxied = format!("{proxied} {extra}");
}
src.replace_range(start..end, &proxied);
}
}
Err(_) => {
// If its not a valid url, replace with empty text
src.replace_range(start..end, "");
}
}
}
(src, links)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
#![allow(clippy::indexing_slicing)]
use super::*;
use pretty_assertions::assert_eq;
#[test]
fn test_basic_markdown() {
let tests: Vec<_> = vec![
(
"headings",
"# h1\n## h2\n### h3\n#### h4\n##### h5\n###### h6",
"<h1>h1</h1>\n<h2>h2</h2>\n<h3>h3</h3>\n<h4>h4</h4>\n<h5>h5</h5>\n<h6>h6</h6>\n"
),
(
"line breaks",
"First\rSecond",
"<p>First\nSecond</p>\n"),
(
"emphasis",
"__bold__ **bold** *italic* ***bold+italic***",
"<p><strong>bold</strong> <strong>bold</strong> <em>italic</em> <em><strong>bold+italic</strong></em></p>\n"
),
(
"blockquotes",
"> #### Hello\n > \n > - Hola\n > - 안영 \n>> Goodbye\n",
"<blockquote>\n<h4>Hello</h4>\n<ul>\n<li>Hola</li>\n<li>안영</li>\n</ul>\n<blockquote>\n<p>Goodbye</p>\n</blockquote>\n</blockquote>\n"
),
(
"lists (ordered, unordered)",
"1. pen\n2. apple\n3. apple pen\n- pen\n- pineapple\n- pineapple pen",
"<ol>\n<li>pen</li>\n<li>apple</li>\n<li>apple pen</li>\n</ol>\n<ul>\n<li>pen</li>\n<li>pineapple</li>\n<li>pineapple pen</li>\n</ul>\n"
),
(
"code and code blocks",
"this is my amazing `code snippet` and my amazing ```code block```",
"<p>this is my amazing <code>code snippet</code> and my amazing <code>code block</code></p>\n"
),
// Links with added nofollow attribute
(
"links",
"[Lemmy](https://join-lemmy.org/ \"Join Lemmy!\")",
"<p><a href=\"https://join-lemmy.org/\" rel=\"nofollow\" title=\"Join Lemmy!\">Lemmy</a></p>\n"
),
// Remote images with proxy
(
"images",
"![My linked image](https://example.com/image.png \"image alt text\")",
"<p><img src=\"https://example.com/image.png\" alt=\"My linked image\" title=\"image alt text\" /></p>\n"
),
// Local images without proxy
(
"images",
"![My linked image](https://lemmy-alpha/image.png \"image alt text\")",
"<p><img src=\"https://lemmy-alpha/image.png\" alt=\"My linked image\" title=\"image alt text\" /></p>\n"
),
// Ensure spoiler plugin is added
(
"basic spoiler",
"::: spoiler click to see more\nhow spicy!\n:::\n",
"<details><summary>click to see more</summary><p>how spicy!\n</p></details>\n"
),
(
"escape html special chars",
"<script>alert('xss');</script> hello &\"",
"<p>&lt;script&gt;alert(‘xss’);&lt;/script&gt; hello &amp;&quot;</p>\n"
)
];
tests.iter().for_each(|&(msg, input, expected)| {
let result = markdown_to_html(input);
assert_eq!(
result, expected,
"Testing {}, with original input '{}'",
msg, input
);
});
}
#[test]
fn test_markdown_proxy_images() {
let tests: Vec<_> =
vec![
(
"remote image proxied",
"![link](http://example.com/image.jpg)",
"![link](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)",
),
(
"local image unproxied",
"![link](http://lemmy-alpha/image.jpg)",
"![link](http://lemmy-alpha/image.jpg)",
),
(
"multiple image links",
"![link](http://example.com/image1.jpg) ![link](http://example.com/image2.jpg)",
"![link](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage1.jpg) ![link](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage2.jpg)",
),
(
"empty link handled",
"![image]()",
"![image]()"
),
(
"empty label handled",
"![](http://example.com/image.jpg)",
"![](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)"
),
(
"invalid image link removed",
"![image](http-not-a-link)",
"![image]()"
),
(
"label with nested markdown handled",
"![a *b* c](http://example.com/image.jpg)",
"![a *b* c](https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Fexample.com%2Fimage.jpg)"
),
(
"custom emoji support",
r#"![party-blob](https://www.hexbear.net/pictrs/image/83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#,
r#"![party-blob](https://lemmy-alpha/api/v3/image_proxy?url=https%3A%2F%2Fwww.hexbear.net%2Fpictrs%2Fimage%2F83405746-0620-4728-9358-5f51b040ffee.gif "emoji party-blob")"#
)
];
tests.iter().for_each(|&(msg, input, expected)| {
let result = markdown_rewrite_image_links(input.to_string());
assert_eq!(
result.0, expected,
"Testing {}, with original input '{}'",
msg, input
);
});
}
#[test]
fn test_sanitize_html() {
let sanitized = sanitize_html("<script>alert('xss');</script> hello &\"'");
let expected = "&lt;script>alert(&#x27;xss&#x27;);&lt;/script> hello &amp;&quot;&#x27;";
assert_eq!(expected, sanitized)
}
}

4
docker/federation/lemmy_epsilon.hjson
View file

@ -10,4 +10,8 @@
database: {
host: postgres_epsilon
}
pictrs: {
api_key: "my-pictrs-key"
image_mode: ProxyAllImages
}
}

4
docker/federation/lemmy_gamma.hjson
View file

@ -10,4 +10,8 @@
database: {
host: postgres_gamma
}
pictrs: {
api_key: "my-pictrs-key"
image_mode: ProxyAllImages
}
}

1
docker/lemmy.hjson
View file

@ -21,6 +21,7 @@
pictrs: {
url: "http://pictrs:8080/"
# api_key: "API_KEY"
image_proxy: true
cache_external_link_previews: true
}

4
migrations/2023-10-24-131607_proxy_links/down.sql Normal file
View file

@ -0,0 +1,4 @@
DROP TABLE remote_image;
ALTER TABLE local_image RENAME TO image_upload;

8
migrations/2023-10-24-131607_proxy_links/up.sql Normal file
View file

@ -0,0 +1,8 @@
CREATE TABLE remote_image (
id serial PRIMARY KEY,
link text NOT NULL UNIQUE,
published timestamptz DEFAULT now() NOT NULL
);
ALTER TABLE image_upload RENAME TO local_image;

3
migrations/2023-10-27-142514_post_url_content_type/down.sql Normal file
View file

@ -0,0 +1,3 @@
ALTER TABLE post
DROP COLUMN url_content_type;

3
migrations/2023-10-27-142514_post_url_content_type/up.sql Normal file
View file

@ -0,0 +1,3 @@
ALTER TABLE post
ADD COLUMN url_content_type text;

2
src/api_routes_http.rs
View file

@ -130,11 +130,13 @@ use lemmy_apub::api::{
search::search,
user_settings_backup::{export_settings, import_settings},
};
use lemmy_routes::images::image_proxy;
use lemmy_utils::rate_limit::RateLimitCell;
pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) {
cfg.service(
web::scope("/api/v3")
.route("/image_proxy", web::get().to(image_proxy))
// Site
.service(
web::scope("/site")

13
src/lib.rs
View file

@ -53,7 +53,7 @@ use lemmy_utils::{
};
use prometheus::default_registry;
use prometheus_metrics::serve_prometheus;
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use reqwest_middleware::ClientBuilder;
use reqwest_tracing::TracingMiddleware;
use serde_json::json;
use std::{env, ops::Deref};
@ -198,15 +198,10 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
startup_server_handle.stop(true).await;
}
// Pictrs cannot use proxy
let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?)
.with(TracingMiddleware::default())
.build();
Some(create_http_server(
federation_config.clone(),
SETTINGS.clone(),
federation_enabled,
pictrs_client,
)?)
} else {
None
@ -272,7 +267,6 @@ fn create_http_server(
federation_config: FederationConfig<LemmyContext>,
settings: Settings,
federation_enabled: bool,
pictrs_client: ClientWithMiddleware,
) -> Result<ServerHandle, LemmyError> {
// this must come before the HttpServer creation
// creates a middleware that populates http metrics for each path, method, and status code
@ -284,6 +278,11 @@ fn create_http_server(
let context: LemmyContext = federation_config.deref().clone();
let rate_limit_cell = federation_config.rate_limit_cell().clone();
// Pictrs cannot use proxy
let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?)
.with(TracingMiddleware::default())
.build();
// Create Http server
let bind = (settings.bind, settings.port);
let server = HttpServer::new(move || {