From 36aaa5064400e794ffc4141dcadca6f75d1978db Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 15 Feb 2023 23:05:14 -0500 Subject: [PATCH] Organize utils into separate files. Fixes #2295 (#2736) * Organize utils into separate files. Fixes #2295 * Moving tests. * Fix test. * Fix test 2 --- Cargo.lock | 1 + crates/api/src/community/ban.rs | 2 +- crates/api/src/lib.rs | 2 +- crates/api/src/local_user/ban_person.rs | 2 +- crates/api/src/local_user/save_settings.rs | 2 +- crates/api_common/Cargo.toml | 3 +- crates/api_common/src/utils.rs | 6 +- crates/api_common/src/websocket/send.rs | 2 +- crates/api_crud/src/comment/create.rs | 2 +- crates/api_crud/src/comment/update.rs | 2 +- crates/api_crud/src/community/create.rs | 5 +- crates/api_crud/src/community/remove.rs | 2 +- crates/api_crud/src/community/update.rs | 2 +- crates/api_crud/src/post/create.rs | 5 +- crates/api_crud/src/post/update.rs | 5 +- crates/api_crud/src/private_message/create.rs | 2 +- crates/api_crud/src/private_message/update.rs | 2 +- crates/api_crud/src/site/create.rs | 2 +- crates/api_crud/src/site/update.rs | 2 +- crates/api_crud/src/user/create.rs | 5 +- .../apub/src/activities/block/block_user.rs | 2 +- crates/apub/src/activities/block/mod.rs | 2 +- .../src/activities/create_or_update/mod.rs | 2 +- crates/apub/src/mentions.rs | 2 +- crates/apub/src/objects/comment.rs | 2 +- crates/apub/src/objects/community.rs | 2 +- crates/apub/src/objects/instance.rs | 6 +- crates/apub/src/objects/person.rs | 6 +- crates/apub/src/objects/post.rs | 6 +- crates/apub/src/objects/private_message.rs | 2 +- crates/apub/src/protocol/objects/group.rs | 2 +- crates/db_schema/src/impls/instance.rs | 3 +- crates/routes/src/feeds.rs | 2 +- crates/utils/src/lib.rs | 2 - crates/utils/src/rate_limit/mod.rs | 16 +- crates/utils/src/test.rs | 98 -------- crates/utils/src/utils.rs | 223 ------------------ crates/utils/src/utils/markdown.rs | 3 + crates/utils/src/utils/mention.rs | 48 ++++ crates/utils/src/utils/mod.rs | 5 + crates/utils/src/utils/slurs.rs | 109 +++++++++ crates/utils/src/utils/time.rs | 12 + crates/utils/src/utils/validation.rs | 131 ++++++++++ 43 files changed, 381 insertions(+), 361 deletions(-) delete mode 100644 crates/utils/src/test.rs delete mode 100644 crates/utils/src/utils.rs create mode 100644 crates/utils/src/utils/markdown.rs create mode 100644 crates/utils/src/utils/mention.rs create mode 100644 crates/utils/src/utils/mod.rs create mode 100644 crates/utils/src/utils/slurs.rs create mode 100644 crates/utils/src/utils/time.rs create mode 100644 crates/utils/src/utils/validation.rs diff --git a/Cargo.lock b/Cargo.lock index 7cea9b571..de31dbf0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2401,6 +2401,7 @@ dependencies = [ "tracing", "tracing-opentelemetry 0.17.4", "url", + "uuid", "webpage", ] diff --git a/crates/api/src/community/ban.rs b/crates/api/src/community/ban.rs index e7962d242..41a8ae5fd 100644 --- a/crates/api/src/community/ban.rs +++ b/crates/api/src/community/ban.rs @@ -19,7 +19,7 @@ use lemmy_db_schema::{ traits::{Bannable, Crud, Followable}, }; use lemmy_db_views_actor::structs::PersonViewSafe; -use lemmy_utils::{error::LemmyError, utils::naive_from_unix, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::time::naive_from_unix, ConnectionId}; #[async_trait::async_trait(?Send)] impl Perform for BanFromCommunity { diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 5904a9b9b..217002ab3 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -2,7 +2,7 @@ use actix_web::web::Data; use captcha::Captcha; use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex}; use lemmy_db_schema::source::local_site::LocalSite; -use lemmy_utils::{error::LemmyError, utils::check_slurs, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs, ConnectionId}; mod comment; mod comment_report; diff --git a/crates/api/src/local_user/ban_person.rs b/crates/api/src/local_user/ban_person.rs index d45528efe..b0686949a 100644 --- a/crates/api/src/local_user/ban_person.rs +++ b/crates/api/src/local_user/ban_person.rs @@ -14,7 +14,7 @@ use lemmy_db_schema::{ traits::Crud, }; use lemmy_db_views_actor::structs::PersonViewSafe; -use lemmy_utils::{error::LemmyError, utils::naive_from_unix, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::time::naive_from_unix, ConnectionId}; #[async_trait::async_trait(?Send)] impl Perform for BanPerson { diff --git a/crates/api/src/local_user/save_settings.rs b/crates/api/src/local_user/save_settings.rs index 6672a67b0..f3f7a8478 100644 --- a/crates/api/src/local_user/save_settings.rs +++ b/crates/api/src/local_user/save_settings.rs @@ -18,7 +18,7 @@ use lemmy_db_schema::{ use lemmy_utils::{ claims::Claims, error::LemmyError, - utils::{is_valid_display_name, is_valid_matrix_id}, + utils::validation::{is_valid_display_name, is_valid_matrix_id}, ConnectionId, }; diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 21eed1c6d..368163d0b 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -47,7 +47,6 @@ tracing-opentelemetry = { workspace = true } actix-ws = { workspace = true } futures = { workspace = true } background-jobs = "0.13.0" - -[dev-dependencies] +uuid = { workspace = true } actix-rt = { workspace = true } reqwest = { workspace = true } diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 3bdd94546..130f6d635 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -39,7 +39,7 @@ use lemmy_utils::{ location_info, rate_limit::RateLimitConfig, settings::structs::Settings, - utils::{build_slur_regex, generate_random_string}, + utils::slurs::build_slur_regex, }; use regex::Regex; use reqwest_middleware::ClientWithMiddleware; @@ -360,7 +360,7 @@ pub async fn send_password_reset_email( settings: &Settings, ) -> Result<(), LemmyError> { // Generate a random token - let token = generate_random_string(); + let token = uuid::Uuid::new_v4().to_string(); // Insert the row let token2 = token.clone(); @@ -386,7 +386,7 @@ pub async fn send_verification_email( let form = EmailVerificationForm { local_user_id: user.local_user.id, email: new_email.to_string(), - verification_token: generate_random_string(), + verification_token: uuid::Uuid::new_v4().to_string(), }; let verify_link = format!( "{}/verify_email/{}", diff --git a/crates/api_common/src/websocket/send.rs b/crates/api_common/src/websocket/send.rs index e1d05202c..44d380d34 100644 --- a/crates/api_common/src/websocket/send.rs +++ b/crates/api_common/src/websocket/send.rs @@ -22,7 +22,7 @@ use lemmy_db_schema::{ }; use lemmy_db_views::structs::{CommentView, LocalUserView, PostView, PrivateMessageView}; use lemmy_db_views_actor::structs::CommunityView; -use lemmy_utils::{error::LemmyError, utils::MentionData, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::mention::MentionData, ConnectionId}; #[tracing::instrument(skip_all)] pub async fn send_post_ws_message( diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index 8c61a0444..d9f4f07cb 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -30,7 +30,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{remove_slurs, scrape_text_for_mentions}, + utils::{mention::scrape_text_for_mentions, slurs::remove_slurs}, ConnectionId, }; diff --git a/crates/api_crud/src/comment/update.rs b/crates/api_crud/src/comment/update.rs index 0cc203a66..f75de78ba 100644 --- a/crates/api_crud/src/comment/update.rs +++ b/crates/api_crud/src/comment/update.rs @@ -26,7 +26,7 @@ use lemmy_db_schema::{ use lemmy_db_views::structs::CommentView; use lemmy_utils::{ error::LemmyError, - utils::{remove_slurs, scrape_text_for_mentions}, + utils::{mention::scrape_text_for_mentions, slurs::remove_slurs}, ConnectionId, }; diff --git a/crates/api_crud/src/community/create.rs b/crates/api_crud/src/community/create.rs index 680d0c7c7..1bf7ba800 100644 --- a/crates/api_crud/src/community/create.rs +++ b/crates/api_crud/src/community/create.rs @@ -34,7 +34,10 @@ use lemmy_db_views::structs::SiteView; use lemmy_db_views_actor::structs::CommunityView; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs, check_slurs_opt, is_valid_actor_name}, + utils::{ + slurs::{check_slurs, check_slurs_opt}, + validation::is_valid_actor_name, + }, ConnectionId, }; diff --git a/crates/api_crud/src/community/remove.rs b/crates/api_crud/src/community/remove.rs index 215b39cf4..80e23bf71 100644 --- a/crates/api_crud/src/community/remove.rs +++ b/crates/api_crud/src/community/remove.rs @@ -13,7 +13,7 @@ use lemmy_db_schema::{ }, traits::Crud, }; -use lemmy_utils::{error::LemmyError, utils::naive_from_unix, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::time::naive_from_unix, ConnectionId}; #[async_trait::async_trait(?Send)] impl PerformCrud for RemoveCommunity { diff --git a/crates/api_crud/src/community/update.rs b/crates/api_crud/src/community/update.rs index 6ab161070..abfa217f9 100644 --- a/crates/api_crud/src/community/update.rs +++ b/crates/api_crud/src/community/update.rs @@ -17,7 +17,7 @@ use lemmy_db_schema::{ utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now}, }; use lemmy_db_views_actor::structs::CommunityModeratorView; -use lemmy_utils::{error::LemmyError, utils::check_slurs_opt, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs_opt, ConnectionId}; #[async_trait::async_trait(?Send)] impl PerformCrud for EditCommunity { diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index af62135e2..655592506 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -29,7 +29,10 @@ use lemmy_db_schema::{ use lemmy_db_views_actor::structs::CommunityView; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs, check_slurs_opt, clean_url_params, is_valid_post_title}, + utils::{ + slurs::{check_slurs, check_slurs_opt}, + validation::{clean_url_params, is_valid_post_title}, + }, ConnectionId, }; use tracing::{warn, Instrument}; diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index 27a4f4dc0..4e43770de 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -18,7 +18,10 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs_opt, clean_url_params, is_valid_post_title}, + utils::{ + slurs::check_slurs_opt, + validation::{clean_url_params, is_valid_post_title}, + }, ConnectionId, }; diff --git a/crates/api_crud/src/private_message/create.rs b/crates/api_crud/src/private_message/create.rs index 6fa5f9134..91556c9d0 100644 --- a/crates/api_crud/src/private_message/create.rs +++ b/crates/api_crud/src/private_message/create.rs @@ -22,7 +22,7 @@ use lemmy_db_schema::{ traits::Crud, }; use lemmy_db_views::structs::LocalUserView; -use lemmy_utils::{error::LemmyError, utils::remove_slurs, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::slurs::remove_slurs, ConnectionId}; #[async_trait::async_trait(?Send)] impl PerformCrud for CreatePrivateMessage { diff --git a/crates/api_crud/src/private_message/update.rs b/crates/api_crud/src/private_message/update.rs index f6f8c8f03..19da0cf55 100644 --- a/crates/api_crud/src/private_message/update.rs +++ b/crates/api_crud/src/private_message/update.rs @@ -14,7 +14,7 @@ use lemmy_db_schema::{ traits::Crud, utils::naive_now, }; -use lemmy_utils::{error::LemmyError, utils::remove_slurs, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::slurs::remove_slurs, ConnectionId}; #[async_trait::async_trait(?Send)] impl PerformCrud for EditPrivateMessage { diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index 8b0b3696f..90b5dd8c9 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -26,7 +26,7 @@ use lemmy_db_schema::{ use lemmy_db_views::structs::SiteView; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs, check_slurs_opt}, + utils::slurs::{check_slurs, check_slurs_opt}, ConnectionId, }; use url::Url; diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index 271a319fa..a169f237d 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -28,7 +28,7 @@ use lemmy_db_schema::{ ListingType, }; use lemmy_db_views::structs::SiteView; -use lemmy_utils::{error::LemmyError, utils::check_slurs_opt, ConnectionId}; +use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs_opt, ConnectionId}; use std::str::FromStr; #[async_trait::async_trait(?Send)] diff --git a/crates/api_crud/src/user/create.rs b/crates/api_crud/src/user/create.rs index a6979382a..c95392001 100644 --- a/crates/api_crud/src/user/create.rs +++ b/crates/api_crud/src/user/create.rs @@ -30,7 +30,10 @@ use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_utils::{ claims::Claims, error::LemmyError, - utils::{check_slurs, check_slurs_opt, is_valid_actor_name}, + utils::{ + slurs::{check_slurs, check_slurs_opt}, + validation::is_valid_actor_name, + }, ConnectionId, }; diff --git a/crates/apub/src/activities/block/block_user.rs b/crates/apub/src/activities/block/block_user.rs index b5f03eaf1..a2f96fd5d 100644 --- a/crates/apub/src/activities/block/block_user.rs +++ b/crates/apub/src/activities/block/block_user.rs @@ -40,7 +40,7 @@ use lemmy_db_schema::{ }, traits::{Bannable, Crud, Followable}, }; -use lemmy_utils::{error::LemmyError, utils::convert_datetime}; +use lemmy_utils::{error::LemmyError, utils::time::convert_datetime}; use url::Url; impl BlockUser { diff --git a/crates/apub/src/activities/block/mod.rs b/crates/apub/src/activities/block/mod.rs index 8adc25012..9a5b9fcb1 100644 --- a/crates/apub/src/activities/block/mod.rs +++ b/crates/apub/src/activities/block/mod.rs @@ -21,7 +21,7 @@ use lemmy_db_schema::{ utils::DbPool, }; use lemmy_db_views::structs::SiteView; -use lemmy_utils::{error::LemmyError, utils::naive_from_unix}; +use lemmy_utils::{error::LemmyError, utils::time::naive_from_unix}; use serde::Deserialize; use url::Url; diff --git a/crates/apub/src/activities/create_or_update/mod.rs b/crates/apub/src/activities/create_or_update/mod.rs index 60ce6ea34..1f8ae79c9 100644 --- a/crates/apub/src/activities/create_or_update/mod.rs +++ b/crates/apub/src/activities/create_or_update/mod.rs @@ -6,7 +6,7 @@ use lemmy_db_schema::{ source::{comment::Comment, post::Post}, traits::Crud, }; -use lemmy_utils::{error::LemmyError, utils::scrape_text_for_mentions}; +use lemmy_utils::{error::LemmyError, utils::mention::scrape_text_for_mentions}; pub mod comment; pub mod post; diff --git a/crates/apub/src/mentions.rs b/crates/apub/src/mentions.rs index 559078942..e23b86344 100644 --- a/crates/apub/src/mentions.rs +++ b/crates/apub/src/mentions.rs @@ -13,7 +13,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{scrape_text_for_mentions, MentionData}, + utils::mention::{scrape_text_for_mentions, MentionData}, }; use serde::{Deserialize, Serialize}; use serde_json::Value; diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index ab11b00d8..6021ff36c 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -33,7 +33,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{convert_datetime, markdown_to_html, remove_slurs}, + utils::{markdown::markdown_to_html, slurs::remove_slurs, time::convert_datetime}, }; use std::ops::Deref; use url::Url; diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index b309d25fc..e23e0743c 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -33,7 +33,7 @@ use lemmy_db_schema::{ use lemmy_db_views_actor::structs::CommunityFollowerView; use lemmy_utils::{ error::LemmyError, - utils::{convert_datetime, markdown_to_html}, + utils::{markdown::markdown_to_html, time::convert_datetime}, }; use std::ops::Deref; use tracing::debug; diff --git a/crates/apub/src/objects/instance.rs b/crates/apub/src/objects/instance.rs index e3cb8b463..fff79463b 100644 --- a/crates/apub/src/objects/instance.rs +++ b/crates/apub/src/objects/instance.rs @@ -30,7 +30,11 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs, check_slurs_opt, convert_datetime, markdown_to_html}, + utils::{ + markdown::markdown_to_html, + slurs::{check_slurs, check_slurs_opt}, + time::convert_datetime, + }, }; use std::ops::Deref; use tracing::debug; diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index 2017b605c..1b7e86d90 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -32,7 +32,11 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs, check_slurs_opt, convert_datetime, markdown_to_html}, + utils::{ + markdown::markdown_to_html, + slurs::{check_slurs, check_slurs_opt}, + time::convert_datetime, + }, }; use std::ops::Deref; use url::Url; diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index e6874f99e..dd7291bd5 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -42,7 +42,11 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs_opt, convert_datetime, markdown_to_html, remove_slurs}, + utils::{ + markdown::markdown_to_html, + slurs::{check_slurs_opt, remove_slurs}, + time::convert_datetime, + }, }; use std::ops::Deref; use url::Url; diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index 4d8c70076..233e8eaa6 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -25,7 +25,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{convert_datetime, markdown_to_html}, + utils::{markdown::markdown_to_html, time::convert_datetime}, }; use std::ops::Deref; use url::Url; diff --git a/crates/apub/src/protocol/objects/group.rs b/crates/apub/src/protocol/objects/group.rs index 5551b1288..a09f9b118 100644 --- a/crates/apub/src/protocol/objects/group.rs +++ b/crates/apub/src/protocol/objects/group.rs @@ -27,7 +27,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs, check_slurs_opt}, + utils::slurs::{check_slurs, check_slurs_opt}, }; use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; diff --git a/crates/db_schema/src/impls/instance.rs b/crates/db_schema/src/impls/instance.rs index 107b28755..32d03e382 100644 --- a/crates/db_schema/src/impls/instance.rs +++ b/crates/db_schema/src/impls/instance.rs @@ -6,7 +6,6 @@ use crate::{ }; use diesel::{dsl::insert_into, result::Error, ExpressionMethods, QueryDsl}; use diesel_async::{AsyncPgConnection, RunQueryDsl}; -use lemmy_utils::utils::generate_domain_url; use url::Url; impl Instance { @@ -28,7 +27,7 @@ impl Instance { Self::create_conn(conn, domain).await } pub async fn create_from_actor_id(pool: &DbPool, actor_id: &Url) -> Result { - let domain = &generate_domain_url(actor_id).expect("actor id missing a domain"); + let domain = actor_id.host_str().expect("actor id missing a domain"); Self::create(pool, domain).await } pub async fn create_conn(conn: &mut AsyncPgConnection, domain: &str) -> Result { diff --git a/crates/routes/src/feeds.rs b/crates/routes/src/feeds.rs index 2e4f815db..5d1f285ca 100644 --- a/crates/routes/src/feeds.rs +++ b/crates/routes/src/feeds.rs @@ -20,7 +20,7 @@ use lemmy_db_views_actor::{ person_mention_view::PersonMentionQuery, structs::{CommentReplyView, PersonMentionView}, }; -use lemmy_utils::{claims::Claims, error::LemmyError, utils::markdown_to_html}; +use lemmy_utils::{claims::Claims, error::LemmyError, utils::markdown::markdown_to_html}; use once_cell::sync::Lazy; use rss::{ extension::dublincore::DublinCoreExtensionBuilder, diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index eeb588439..48d166152 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -11,8 +11,6 @@ pub mod settings; pub mod claims; pub mod error; pub mod request; -#[cfg(test)] -mod test; pub mod utils; pub mod version; diff --git a/crates/utils/src/rate_limit/mod.rs b/crates/utils/src/rate_limit/mod.rs index 6dc9dcbef..b7000e3e6 100644 --- a/crates/utils/src/rate_limit/mod.rs +++ b/crates/utils/src/rate_limit/mod.rs @@ -1,5 +1,5 @@ -use crate::{error::LemmyError, utils::get_ip, IpAddr}; -use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; +use crate::{error::LemmyError, IpAddr}; +use actix_web::dev::{ConnectionInfo, Service, ServiceRequest, ServiceResponse, Transform}; use futures::future::{ok, Ready}; use rate_limiter::{RateLimitStorage, RateLimitType}; use serde::{Deserialize, Serialize}; @@ -220,3 +220,15 @@ where }) } } + +fn get_ip(conn_info: &ConnectionInfo) -> IpAddr { + IpAddr( + conn_info + .realip_remote_addr() + .unwrap_or("127.0.0.1:12345") + .split(':') + .next() + .unwrap_or("127.0.0.1") + .to_string(), + ) +} diff --git a/crates/utils/src/test.rs b/crates/utils/src/test.rs deleted file mode 100644 index 07c35d152..000000000 --- a/crates/utils/src/test.rs +++ /dev/null @@ -1,98 +0,0 @@ -use crate::utils::{ - is_valid_actor_name, - is_valid_display_name, - is_valid_matrix_id, - is_valid_post_title, - remove_slurs, - scrape_text_for_mentions, - slur_check, - slurs_vec_to_str, -}; -use regex::RegexBuilder; - -#[test] -fn test_mentions_regex() { - let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)"; - let mentions = scrape_text_for_mentions(text); - - assert_eq!(mentions[0].name, "tedu".to_string()); - assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string()); - assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string()); -} - -#[test] -fn test_valid_actor_name() { - let actor_name_max_length = 20; - assert!(is_valid_actor_name("Hello_98", actor_name_max_length)); - assert!(is_valid_actor_name("ten", actor_name_max_length)); - assert!(!is_valid_actor_name("Hello-98", actor_name_max_length)); - assert!(!is_valid_actor_name("a", actor_name_max_length)); - assert!(!is_valid_actor_name("", actor_name_max_length)); -} - -#[test] -fn test_valid_display_name() { - let actor_name_max_length = 20; - assert!(is_valid_display_name("hello @there", actor_name_max_length)); - assert!(!is_valid_display_name( - "@hello there", - actor_name_max_length - )); - - // Make sure zero-space with an @ doesn't work - assert!(!is_valid_display_name( - &format!("{}@my name is", '\u{200b}'), - actor_name_max_length - )); -} - -#[test] -fn test_valid_post_title() { - assert!(is_valid_post_title("Post Title")); - assert!(is_valid_post_title(" POST TITLE 😃😃😃😃😃")); - assert!(!is_valid_post_title("\n \n \n \n ")); // tabs/spaces/newlines -} - -#[test] -fn test_valid_matrix_id() { - assert!(is_valid_matrix_id("@dess:matrix.org")); - assert!(!is_valid_matrix_id("dess:matrix.org")); - assert!(!is_valid_matrix_id(" @dess:matrix.org")); - assert!(!is_valid_matrix_id("@dess:matrix.org t")); -} - -#[test] -fn test_slur_filter() { - let slur_regex = Some(RegexBuilder::new(r"(fag(g|got|tard)?\b|cock\s?sucker(s|ing)?|ni((g{2,}|q)+|[gq]{2,})[e3r]+(s|z)?|mudslime?s?|kikes?|\bspi(c|k)s?\b|\bchinks?|gooks?|bitch(es|ing|y)?|whor(es?|ing)|\btr(a|@)nn?(y|ies?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap()); - let test = - "faggot test kike tranny cocksucker retardeds. Capitalized Niggerz. This is a bunch of other safe text."; - let slur_free = "No slurs here"; - assert_eq!( - remove_slurs(test, &slur_regex), - "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text." - .to_string() - ); - - let has_slurs_vec = vec![ - "Niggerz", - "cocksucker", - "faggot", - "kike", - "retardeds", - "tranny", - ]; - let has_slurs_err_str = "No slurs - Niggerz, cocksucker, faggot, kike, retardeds, tranny"; - - assert_eq!(slur_check(test, &slur_regex), Err(has_slurs_vec)); - assert_eq!(slur_check(slur_free, &slur_regex), Ok(())); - if let Err(slur_vec) = slur_check(test, &slur_regex) { - assert_eq!(&slurs_vec_to_str(&slur_vec), has_slurs_err_str); - } -} - -// These helped with testing -// #[test] -// fn test_send_email() { -// let result = send_email("not a subject", "test_email@gmail.com", "ur user", "

HI there

"); -// assert!(result.is_ok()); -// } diff --git a/crates/utils/src/utils.rs b/crates/utils/src/utils.rs deleted file mode 100644 index f3c4289f3..000000000 --- a/crates/utils/src/utils.rs +++ /dev/null @@ -1,223 +0,0 @@ -use crate::{error::LemmyError, location_info, IpAddr}; -use actix_web::dev::ConnectionInfo; -use anyhow::Context; -use chrono::{DateTime, FixedOffset, NaiveDateTime}; -use itertools::Itertools; -use once_cell::sync::Lazy; -use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use regex::{Regex, RegexBuilder}; -use url::Url; - -static MENTIONS_REGEX: Lazy = Lazy::new(|| { - Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").expect("compile regex") -}); -static VALID_ACTOR_NAME_REGEX: Lazy = - Lazy::new(|| Regex::new(r"^[a-zA-Z0-9_]{3,}$").expect("compile regex")); -static VALID_POST_TITLE_REGEX: Lazy = - Lazy::new(|| Regex::new(r".*\S{3,}.*").expect("compile regex")); -static VALID_MATRIX_ID_REGEX: Lazy = Lazy::new(|| { - Regex::new(r"^@[A-Za-z0-9._=-]+:[A-Za-z0-9.-]+\.[A-Za-z]{2,}$").expect("compile regex") -}); -// taken from https://en.wikipedia.org/wiki/UTM_parameters -static CLEAN_URL_PARAMS_REGEX: Lazy = Lazy::new(|| { - Regex::new(r"^utm_source|utm_medium|utm_campaign|utm_term|utm_content|gclid|gclsrc|dclid|fbclid$") - .expect("compile regex") -}); - -pub fn naive_from_unix(time: i64) -> NaiveDateTime { - NaiveDateTime::from_timestamp_opt(time, 0).expect("convert datetime") -} - -pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime { - DateTime::::from_utc( - datetime, - FixedOffset::east_opt(0).expect("create fixed offset"), - ) -} - -pub fn remove_slurs(test: &str, slur_regex: &Option) -> String { - if let Some(slur_regex) = slur_regex { - slur_regex.replace_all(test, "*removed*").to_string() - } else { - test.to_string() - } -} - -pub(crate) fn slur_check<'a>( - test: &'a str, - slur_regex: &'a Option, -) -> Result<(), Vec<&'a str>> { - if let Some(slur_regex) = slur_regex { - let mut matches: Vec<&str> = slur_regex.find_iter(test).map(|mat| mat.as_str()).collect(); - - // Unique - matches.sort_unstable(); - matches.dedup(); - - if matches.is_empty() { - Ok(()) - } else { - Err(matches) - } - } else { - Ok(()) - } -} - -pub fn build_slur_regex(regex_str: Option<&str>) -> Option { - regex_str.map(|slurs| { - RegexBuilder::new(slurs) - .case_insensitive(true) - .build() - .expect("compile regex") - }) -} - -pub fn check_slurs(text: &str, slur_regex: &Option) -> Result<(), LemmyError> { - if let Err(slurs) = slur_check(text, slur_regex) { - Err(LemmyError::from_error_message( - anyhow::anyhow!("{}", slurs_vec_to_str(&slurs)), - "slurs", - )) - } else { - Ok(()) - } -} - -pub fn check_slurs_opt( - text: &Option, - slur_regex: &Option, -) -> Result<(), LemmyError> { - match text { - Some(t) => check_slurs(t, slur_regex), - None => Ok(()), - } -} - -pub(crate) fn slurs_vec_to_str(slurs: &[&str]) -> String { - let start = "No slurs - "; - let combined = &slurs.join(", "); - [start, combined].concat() -} - -pub fn generate_random_string() -> String { - thread_rng() - .sample_iter(&Alphanumeric) - .map(char::from) - .take(30) - .collect() -} - -pub fn markdown_to_html(text: &str) -> String { - comrak::markdown_to_html(text, &comrak::ComrakOptions::default()) -} - -// TODO nothing is done with community / group webfingers yet, so just ignore those for now -#[derive(Clone, PartialEq, Eq, Hash)] -pub struct MentionData { - pub name: String, - pub domain: String, -} - -impl MentionData { - pub fn is_local(&self, hostname: &str) -> bool { - hostname.eq(&self.domain) - } - pub fn full_name(&self) -> String { - format!("@{}@{}", &self.name, &self.domain) - } -} - -pub fn scrape_text_for_mentions(text: &str) -> Vec { - let mut out: Vec = Vec::new(); - for caps in MENTIONS_REGEX.captures_iter(text) { - out.push(MentionData { - name: caps["name"].to_string(), - domain: caps["domain"].to_string(), - }); - } - out.into_iter().unique().collect() -} - -fn has_newline(name: &str) -> bool { - name.contains('\n') -} - -pub fn is_valid_actor_name(name: &str, actor_name_max_length: usize) -> bool { - name.chars().count() <= actor_name_max_length - && VALID_ACTOR_NAME_REGEX.is_match(name) - && !has_newline(name) -} - -// Can't do a regex here, reverse lookarounds not supported -pub fn is_valid_display_name(name: &str, actor_name_max_length: usize) -> bool { - !name.starts_with('@') - && !name.starts_with('\u{200b}') - && name.chars().count() >= 3 - && name.chars().count() <= actor_name_max_length - && !has_newline(name) -} - -pub fn is_valid_matrix_id(matrix_id: &str) -> bool { - VALID_MATRIX_ID_REGEX.is_match(matrix_id) && !has_newline(matrix_id) -} - -pub fn is_valid_post_title(title: &str) -> bool { - VALID_POST_TITLE_REGEX.is_match(title) && !has_newline(title) -} - -pub fn get_ip(conn_info: &ConnectionInfo) -> IpAddr { - IpAddr( - conn_info - .realip_remote_addr() - .unwrap_or("127.0.0.1:12345") - .split(':') - .next() - .unwrap_or("127.0.0.1") - .to_string(), - ) -} - -pub fn clean_url_params(url: &Url) -> Url { - let mut url_out = url.clone(); - if url.query().is_some() { - let new_query = url - .query_pairs() - .filter(|q| !CLEAN_URL_PARAMS_REGEX.is_match(&q.0)) - .map(|q| format!("{}={}", q.0, q.1)) - .join("&"); - url_out.set_query(Some(&new_query)); - } - url_out -} - -pub fn generate_domain_url(actor_id: &Url) -> Result { - Ok(actor_id.host_str().context(location_info!())?.to_string()) -} - -#[cfg(test)] -mod tests { - use crate::utils::{clean_url_params, is_valid_post_title}; - use url::Url; - - #[test] - fn test_clean_url_params() { - let url = Url::parse("https://example.com/path/123?utm_content=buffercf3b2&utm_medium=social&username=randomuser&id=123").unwrap(); - let cleaned = clean_url_params(&url); - let expected = Url::parse("https://example.com/path/123?username=randomuser&id=123").unwrap(); - assert_eq!(expected.to_string(), cleaned.to_string()); - - let url = Url::parse("https://example.com/path/123").unwrap(); - let cleaned = clean_url_params(&url); - assert_eq!(url.to_string(), cleaned.to_string()); - } - - #[test] - fn regex_checks() { - assert!(!is_valid_post_title("hi")); - assert!(is_valid_post_title("him")); - assert!(!is_valid_post_title("n\n\n\n\nanother")); - assert!(!is_valid_post_title("hello there!\n this is a test.")); - assert!(is_valid_post_title("hello there! this is a test.")); - } -} diff --git a/crates/utils/src/utils/markdown.rs b/crates/utils/src/utils/markdown.rs new file mode 100644 index 000000000..a5ee53571 --- /dev/null +++ b/crates/utils/src/utils/markdown.rs @@ -0,0 +1,3 @@ +pub fn markdown_to_html(text: &str) -> String { + comrak::markdown_to_html(text, &comrak::ComrakOptions::default()) +} diff --git a/crates/utils/src/utils/mention.rs b/crates/utils/src/utils/mention.rs new file mode 100644 index 000000000..d0383514c --- /dev/null +++ b/crates/utils/src/utils/mention.rs @@ -0,0 +1,48 @@ +use itertools::Itertools; +use once_cell::sync::Lazy; +use regex::Regex; + +static MENTIONS_REGEX: Lazy = Lazy::new(|| { + Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").expect("compile regex") +}); +// TODO nothing is done with community / group webfingers yet, so just ignore those for now +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct MentionData { + pub name: String, + pub domain: String, +} + +impl MentionData { + pub fn is_local(&self, hostname: &str) -> bool { + hostname.eq(&self.domain) + } + pub fn full_name(&self) -> String { + format!("@{}@{}", &self.name, &self.domain) + } +} + +pub fn scrape_text_for_mentions(text: &str) -> Vec { + let mut out: Vec = Vec::new(); + for caps in MENTIONS_REGEX.captures_iter(text) { + out.push(MentionData { + name: caps["name"].to_string(), + domain: caps["domain"].to_string(), + }); + } + out.into_iter().unique().collect() +} + +#[cfg(test)] +mod test { + use crate::utils::mention::scrape_text_for_mentions; + + #[test] + fn test_mentions_regex() { + let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)"; + let mentions = scrape_text_for_mentions(text); + + assert_eq!(mentions[0].name, "tedu".to_string()); + assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string()); + assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string()); + } +} diff --git a/crates/utils/src/utils/mod.rs b/crates/utils/src/utils/mod.rs new file mode 100644 index 000000000..04be57d34 --- /dev/null +++ b/crates/utils/src/utils/mod.rs @@ -0,0 +1,5 @@ +pub mod markdown; +pub mod mention; +pub mod slurs; +pub mod time; +pub mod validation; diff --git a/crates/utils/src/utils/slurs.rs b/crates/utils/src/utils/slurs.rs new file mode 100644 index 000000000..b92650ea3 --- /dev/null +++ b/crates/utils/src/utils/slurs.rs @@ -0,0 +1,109 @@ +use crate::error::LemmyError; +use regex::{Regex, RegexBuilder}; + +pub fn remove_slurs(test: &str, slur_regex: &Option) -> String { + if let Some(slur_regex) = slur_regex { + slur_regex.replace_all(test, "*removed*").to_string() + } else { + test.to_string() + } +} + +pub(crate) fn slur_check<'a>( + test: &'a str, + slur_regex: &'a Option, +) -> Result<(), Vec<&'a str>> { + if let Some(slur_regex) = slur_regex { + let mut matches: Vec<&str> = slur_regex.find_iter(test).map(|mat| mat.as_str()).collect(); + + // Unique + matches.sort_unstable(); + matches.dedup(); + + if matches.is_empty() { + Ok(()) + } else { + Err(matches) + } + } else { + Ok(()) + } +} + +pub fn build_slur_regex(regex_str: Option<&str>) -> Option { + regex_str.map(|slurs| { + RegexBuilder::new(slurs) + .case_insensitive(true) + .build() + .expect("compile regex") + }) +} + +pub fn check_slurs(text: &str, slur_regex: &Option) -> Result<(), LemmyError> { + if let Err(slurs) = slur_check(text, slur_regex) { + Err(LemmyError::from_error_message( + anyhow::anyhow!("{}", slurs_vec_to_str(&slurs)), + "slurs", + )) + } else { + Ok(()) + } +} + +pub fn check_slurs_opt( + text: &Option, + slur_regex: &Option, +) -> Result<(), LemmyError> { + match text { + Some(t) => check_slurs(t, slur_regex), + None => Ok(()), + } +} + +pub(crate) fn slurs_vec_to_str(slurs: &[&str]) -> String { + let start = "No slurs - "; + let combined = &slurs.join(", "); + [start, combined].concat() +} + +#[cfg(test)] +mod test { + use crate::utils::slurs::{remove_slurs, slur_check, slurs_vec_to_str}; + use regex::RegexBuilder; + + #[test] + fn test_slur_filter() { + let slur_regex = Some(RegexBuilder::new(r"(fag(g|got|tard)?\b|cock\s?sucker(s|ing)?|ni((g{2,}|q)+|[gq]{2,})[e3r]+(s|z)?|mudslime?s?|kikes?|\bspi(c|k)s?\b|\bchinks?|gooks?|bitch(es|ing|y)?|whor(es?|ing)|\btr(a|@)nn?(y|ies?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap()); + let test = + "faggot test kike tranny cocksucker retardeds. Capitalized Niggerz. This is a bunch of other safe text."; + let slur_free = "No slurs here"; + assert_eq!( + remove_slurs(test, &slur_regex), + "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text." + .to_string() + ); + + let has_slurs_vec = vec![ + "Niggerz", + "cocksucker", + "faggot", + "kike", + "retardeds", + "tranny", + ]; + let has_slurs_err_str = "No slurs - Niggerz, cocksucker, faggot, kike, retardeds, tranny"; + + assert_eq!(slur_check(test, &slur_regex), Err(has_slurs_vec)); + assert_eq!(slur_check(slur_free, &slur_regex), Ok(())); + if let Err(slur_vec) = slur_check(test, &slur_regex) { + assert_eq!(&slurs_vec_to_str(&slur_vec), has_slurs_err_str); + } + } + + // These helped with testing + // #[test] + // fn test_send_email() { + // let result = send_email("not a subject", "test_email@gmail.com", "ur user", "

HI there

"); + // assert!(result.is_ok()); + // } +} diff --git a/crates/utils/src/utils/time.rs b/crates/utils/src/utils/time.rs new file mode 100644 index 000000000..c1e167651 --- /dev/null +++ b/crates/utils/src/utils/time.rs @@ -0,0 +1,12 @@ +use chrono::{DateTime, FixedOffset, NaiveDateTime}; + +pub fn naive_from_unix(time: i64) -> NaiveDateTime { + NaiveDateTime::from_timestamp_opt(time, 0).expect("convert datetime") +} + +pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime { + DateTime::::from_utc( + datetime, + FixedOffset::east_opt(0).expect("create fixed offset"), + ) +} diff --git a/crates/utils/src/utils/validation.rs b/crates/utils/src/utils/validation.rs new file mode 100644 index 000000000..43f3cb35f --- /dev/null +++ b/crates/utils/src/utils/validation.rs @@ -0,0 +1,131 @@ +use itertools::Itertools; +use once_cell::sync::Lazy; +use regex::Regex; +use url::Url; + +static VALID_ACTOR_NAME_REGEX: Lazy = + Lazy::new(|| Regex::new(r"^[a-zA-Z0-9_]{3,}$").expect("compile regex")); +static VALID_POST_TITLE_REGEX: Lazy = + Lazy::new(|| Regex::new(r".*\S{3,}.*").expect("compile regex")); +static VALID_MATRIX_ID_REGEX: Lazy = Lazy::new(|| { + Regex::new(r"^@[A-Za-z0-9._=-]+:[A-Za-z0-9.-]+\.[A-Za-z]{2,}$").expect("compile regex") +}); +// taken from https://en.wikipedia.org/wiki/UTM_parameters +static CLEAN_URL_PARAMS_REGEX: Lazy = Lazy::new(|| { + Regex::new(r"^utm_source|utm_medium|utm_campaign|utm_term|utm_content|gclid|gclsrc|dclid|fbclid$") + .expect("compile regex") +}); + +fn has_newline(name: &str) -> bool { + name.contains('\n') +} + +pub fn is_valid_actor_name(name: &str, actor_name_max_length: usize) -> bool { + name.chars().count() <= actor_name_max_length + && VALID_ACTOR_NAME_REGEX.is_match(name) + && !has_newline(name) +} + +// Can't do a regex here, reverse lookarounds not supported +pub fn is_valid_display_name(name: &str, actor_name_max_length: usize) -> bool { + !name.starts_with('@') + && !name.starts_with('\u{200b}') + && name.chars().count() >= 3 + && name.chars().count() <= actor_name_max_length + && !has_newline(name) +} + +pub fn is_valid_matrix_id(matrix_id: &str) -> bool { + VALID_MATRIX_ID_REGEX.is_match(matrix_id) && !has_newline(matrix_id) +} + +pub fn is_valid_post_title(title: &str) -> bool { + VALID_POST_TITLE_REGEX.is_match(title) && !has_newline(title) +} + +pub fn clean_url_params(url: &Url) -> Url { + let mut url_out = url.clone(); + if url.query().is_some() { + let new_query = url + .query_pairs() + .filter(|q| !CLEAN_URL_PARAMS_REGEX.is_match(&q.0)) + .map(|q| format!("{}={}", q.0, q.1)) + .join("&"); + url_out.set_query(Some(&new_query)); + } + url_out +} + +#[cfg(test)] +mod tests { + use crate::utils::validation::{ + clean_url_params, + is_valid_actor_name, + is_valid_display_name, + is_valid_matrix_id, + is_valid_post_title, + }; + use url::Url; + + #[test] + fn test_clean_url_params() { + let url = Url::parse("https://example.com/path/123?utm_content=buffercf3b2&utm_medium=social&username=randomuser&id=123").unwrap(); + let cleaned = clean_url_params(&url); + let expected = Url::parse("https://example.com/path/123?username=randomuser&id=123").unwrap(); + assert_eq!(expected.to_string(), cleaned.to_string()); + + let url = Url::parse("https://example.com/path/123").unwrap(); + let cleaned = clean_url_params(&url); + assert_eq!(url.to_string(), cleaned.to_string()); + } + + #[test] + fn regex_checks() { + assert!(!is_valid_post_title("hi")); + assert!(is_valid_post_title("him")); + assert!(!is_valid_post_title("n\n\n\n\nanother")); + assert!(!is_valid_post_title("hello there!\n this is a test.")); + assert!(is_valid_post_title("hello there! this is a test.")); + } + + #[test] + fn test_valid_actor_name() { + let actor_name_max_length = 20; + assert!(is_valid_actor_name("Hello_98", actor_name_max_length)); + assert!(is_valid_actor_name("ten", actor_name_max_length)); + assert!(!is_valid_actor_name("Hello-98", actor_name_max_length)); + assert!(!is_valid_actor_name("a", actor_name_max_length)); + assert!(!is_valid_actor_name("", actor_name_max_length)); + } + + #[test] + fn test_valid_display_name() { + let actor_name_max_length = 20; + assert!(is_valid_display_name("hello @there", actor_name_max_length)); + assert!(!is_valid_display_name( + "@hello there", + actor_name_max_length + )); + + // Make sure zero-space with an @ doesn't work + assert!(!is_valid_display_name( + &format!("{}@my name is", '\u{200b}'), + actor_name_max_length + )); + } + + #[test] + fn test_valid_post_title() { + assert!(is_valid_post_title("Post Title")); + assert!(is_valid_post_title(" POST TITLE 😃😃😃😃😃")); + assert!(!is_valid_post_title("\n \n \n \n ")); // tabs/spaces/newlines + } + + #[test] + fn test_valid_matrix_id() { + assert!(is_valid_matrix_id("@dess:matrix.org")); + assert!(!is_valid_matrix_id("dess:matrix.org")); + assert!(!is_valid_matrix_id(" @dess:matrix.org")); + assert!(!is_valid_matrix_id("@dess:matrix.org t")); + } +}