From 9d4973829bb2ff670401ab0d8b7810db103df7b9 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 18 Sep 2020 16:37:46 +0200 Subject: [PATCH 1/4] Add integration test to ensure that signatures are verified --- lemmy_utils/src/lib.rs | 2 +- src/apub/activity_queue.rs | 1 - src/lib.rs | 14 ++-- src/main.rs | 2 +- src/websocket/chat_server.rs | 7 +- test.sh | 2 +- tests/integration_test.rs | 126 +++++++++++++++++++++++++++++++++++ 7 files changed, 137 insertions(+), 17 deletions(-) create mode 100644 tests/integration_test.rs diff --git a/lemmy_utils/src/lib.rs b/lemmy_utils/src/lib.rs index 9cc1fe025..247705e42 100644 --- a/lemmy_utils/src/lib.rs +++ b/lemmy_utils/src/lib.rs @@ -57,7 +57,7 @@ impl APIError { #[derive(Debug)] pub struct LemmyError { - inner: anyhow::Error, + pub inner: anyhow::Error, } impl From for LemmyError diff --git a/src/apub/activity_queue.rs b/src/apub/activity_queue.rs index 008846bf3..a48cf5bd2 100644 --- a/src/apub/activity_queue.rs +++ b/src/apub/activity_queue.rs @@ -79,7 +79,6 @@ impl ActixJob for SendActivityTask { .post(to_url.as_str()) .header("Content-Type", "application/json"); - // TODO: i believe we have to do the signing in here because it is only valid for a few seconds let signed = sign( request, self.activity.clone(), diff --git a/src/lib.rs b/src/lib.rs index 2bab3552a..5d5f00b0d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,7 +50,7 @@ pub struct LemmyContext { } impl LemmyContext { - pub fn create( + pub fn new( pool: DbPool, chat_server: Addr, client: Client, @@ -79,12 +79,12 @@ impl LemmyContext { impl Clone for LemmyContext { fn clone(&self) -> Self { - LemmyContext { - pool: self.pool.clone(), - chat_server: self.chat_server.clone(), - client: self.client.clone(), - activity_queue: self.activity_queue.clone(), - } + LemmyContext::new( + self.pool.clone(), + self.chat_server.clone(), + self.client.clone(), + self.activity_queue.clone(), + ) } } diff --git a/src/main.rs b/src/main.rs index 800196cfe..d14e006e2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -84,7 +84,7 @@ async fn main() -> Result<(), LemmyError> { // Create Http server with websocket support HttpServer::new(move || { - let context = LemmyContext::create( + let context = LemmyContext::new( pool.clone(), chat_server.to_owned(), Client::default(), diff --git a/src/websocket/chat_server.rs b/src/websocket/chat_server.rs index 0c4dfb0ba..611f44f97 100644 --- a/src/websocket/chat_server.rs +++ b/src/websocket/chat_server.rs @@ -361,12 +361,7 @@ impl ChatServer { let user_operation: UserOperation = UserOperation::from_str(&op)?; - let context = LemmyContext { - pool, - chat_server: addr, - client, - activity_queue, - }; + let context = LemmyContext::new(pool, addr, client, activity_queue); let args = Args { context, rate_limiter, diff --git a/test.sh b/test.sh index beb499bdf..801e95a41 100755 --- a/test.sh +++ b/test.sh @@ -2,4 +2,4 @@ export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy diesel migration run export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy -RUST_TEST_THREADS=1 cargo test --workspace --no-fail-fast +RUST_TEST_THREADS=1 RUST_BACKTRACE=1 cargo test -j8 --no-fail-fast -- --nocapture diff --git a/tests/integration_test.rs b/tests/integration_test.rs new file mode 100644 index 000000000..a127ad42f --- /dev/null +++ b/tests/integration_test.rs @@ -0,0 +1,126 @@ +extern crate lemmy_server; + +use activitystreams::{ + activity::{kind::CreateType, ActorAndObject}, + base::{BaseExt, ExtendsExt}, + object::{Note, ObjectExt}, +}; +use actix::prelude::*; +use actix_web::{test::TestRequest, web}; +use chrono::Utc; +use diesel::{ + r2d2::{ConnectionManager, Pool}, + PgConnection, +}; +use http_signature_normalization_actix::PrepareVerifyError; +use lemmy_db::{ + user::{User_, *}, + Crud, + ListingType, + SortType, +}; +use lemmy_rate_limit::{rate_limiter::RateLimiter, RateLimit}; +use lemmy_server::{ + apub::{ + activity_queue::create_activity_queue, + inbox::shared_inbox::{shared_inbox, ValidTypes}, + }, + websocket::chat_server::ChatServer, + LemmyContext, +}; +use lemmy_utils::{apub::generate_actor_keypair, settings::Settings}; +use reqwest::Client; +use std::sync::Arc; +use tokio::sync::Mutex; +use url::Url; + +fn create_context() -> LemmyContext { + let settings = Settings::get(); + let db_url = settings.get_database_url(); + let manager = ConnectionManager::::new(&db_url); + let pool = Pool::builder() + .max_size(settings.database.pool_size) + .build(manager) + .unwrap(); + let rate_limiter = RateLimit { + rate_limiter: Arc::new(Mutex::new(RateLimiter::default())), + }; + let activity_queue = create_activity_queue(); + let chat_server = ChatServer::startup( + pool.clone(), + rate_limiter.clone(), + Client::default(), + activity_queue.clone(), + ) + .start(); + LemmyContext::new( + pool, + chat_server, + Client::default(), + create_activity_queue(), + ) +} + +fn create_user(conn: &PgConnection) -> User_ { + let user_keypair = generate_actor_keypair().unwrap(); + let new_user = UserForm { + name: "integration_user_1".into(), + preferred_username: None, + password_encrypted: "nope".into(), + email: None, + matrix_user_id: None, + avatar: None, + banner: None, + admin: false, + banned: false, + updated: None, + show_nsfw: false, + theme: "darkly".into(), + default_sort_type: SortType::Hot as i16, + default_listing_type: ListingType::Subscribed as i16, + lang: "browser".into(), + show_avatars: true, + send_notifications_to_email: false, + actor_id: Some("http://localhost:8536/u/integration_user_1".to_string()), + bio: None, + local: true, + private_key: Some(user_keypair.private_key), + public_key: Some(user_keypair.public_key), + last_refreshed_at: None, + }; + + User_::create(&conn, &new_user).unwrap() +} + +fn create_activity(user_id: String) -> web::Json> { + let mut activity = + ActorAndObject::::new(user_id, Note::new().into_any_base().unwrap()); + activity + .set_id(Url::parse("http://localhost:8536/create/1").unwrap()) + .set_many_ccs(vec![Url::parse("http://localhost:8536/c/main").unwrap()]); + let activity = serde_json::to_value(&activity).unwrap(); + let activity: ActorAndObject = serde_json::from_value(activity).unwrap(); + web::Json(activity) +} + +#[actix_rt::test] +async fn test_expired_signature() { + let time1 = Utc::now().timestamp(); + let time2 = Utc::now().timestamp(); + let signature = format!( + r#"keyId="my-key-id",algorithm="hs2019",created="{}",expires="{}",headers="(request-target) (created) (expires) date content-type",signature="blah blah blah""#, + time1, time2 + ); + let request = TestRequest::post() + .uri("http://localhost:8536/inbox") + .header("Signature", signature) + .to_http_request(); + let context = create_context(); + let user = create_user(&context.pool().get().unwrap()); + let activity = create_activity(user.actor_id); + let response = shared_inbox(request, activity, web::Data::new(context)).await; + assert_eq!( + format!("{}", response.err().unwrap()), + format!("{}", PrepareVerifyError::Expired) + ); +} From db3dcc6fdce6f7fc5b7324fb4b50a377ef6472d7 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 18 Sep 2020 17:45:50 +0200 Subject: [PATCH 2/4] Add tests for community_inbox and user_inbox --- tests/integration_test.rs | 109 ++++++++++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 17 deletions(-) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index a127ad42f..382bd65da 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,12 +1,15 @@ extern crate lemmy_server; use activitystreams::{ - activity::{kind::CreateType, ActorAndObject}, + activity::{ + kind::{CreateType, FollowType}, + ActorAndObject, + }, base::{BaseExt, ExtendsExt}, object::{Note, ObjectExt}, }; use actix::prelude::*; -use actix_web::{test::TestRequest, web}; +use actix_web::{test::TestRequest, web, web::Path, HttpRequest}; use chrono::Utc; use diesel::{ r2d2::{ConnectionManager, Pool}, @@ -14,6 +17,7 @@ use diesel::{ }; use http_signature_normalization_actix::PrepareVerifyError; use lemmy_db::{ + community::{Community, CommunityForm}, user::{User_, *}, Crud, ListingType, @@ -23,13 +27,21 @@ use lemmy_rate_limit::{rate_limiter::RateLimiter, RateLimit}; use lemmy_server::{ apub::{ activity_queue::create_activity_queue, - inbox::shared_inbox::{shared_inbox, ValidTypes}, + inbox::{ + community_inbox, + community_inbox::community_inbox, + shared_inbox, + shared_inbox::shared_inbox, + user_inbox, + user_inbox::user_inbox, + }, }, websocket::chat_server::ChatServer, LemmyContext, }; use lemmy_utils::{apub::generate_actor_keypair, settings::Settings}; use reqwest::Client; +use serde::{Deserialize, Serialize}; use std::sync::Arc; use tokio::sync::Mutex; use url::Url; @@ -61,10 +73,10 @@ fn create_context() -> LemmyContext { ) } -fn create_user(conn: &PgConnection) -> User_ { +fn create_user(conn: &PgConnection, name: &str) -> User_ { let user_keypair = generate_actor_keypair().unwrap(); let new_user = UserForm { - name: "integration_user_1".into(), + name: name.into(), preferred_username: None, password_encrypted: "nope".into(), email: None, @@ -81,7 +93,7 @@ fn create_user(conn: &PgConnection) -> User_ { lang: "browser".into(), show_avatars: true, send_notifications_to_email: false, - actor_id: Some("http://localhost:8536/u/integration_user_1".to_string()), + actor_id: Some(format!("http://localhost:8536/u/{}", name).to_string()), bio: None, local: true, private_key: Some(user_keypair.private_key), @@ -92,35 +104,98 @@ fn create_user(conn: &PgConnection) -> User_ { User_::create(&conn, &new_user).unwrap() } -fn create_activity(user_id: String) -> web::Json> { - let mut activity = - ActorAndObject::::new(user_id, Note::new().into_any_base().unwrap()); +fn create_community(conn: &PgConnection, creator_id: i32) -> Community { + let new_community = CommunityForm { + name: "test_community".into(), + creator_id, + title: "test_community".to_owned(), + description: None, + category_id: 1, + nsfw: false, + removed: None, + deleted: None, + updated: None, + actor_id: None, + local: true, + private_key: None, + public_key: None, + last_refreshed_at: None, + published: None, + icon: None, + banner: None, + }; + Community::create(&conn, &new_community).unwrap() +} +fn create_activity<'a, Activity, Return>(user_id: String) -> web::Json +where + for<'de> Return: Deserialize<'de> + 'a, + Activity: std::default::Default + Serialize, +{ + let mut activity = ActorAndObject::::new(user_id, Note::new().into_any_base().unwrap()); activity .set_id(Url::parse("http://localhost:8536/create/1").unwrap()) .set_many_ccs(vec![Url::parse("http://localhost:8536/c/main").unwrap()]); let activity = serde_json::to_value(&activity).unwrap(); - let activity: ActorAndObject = serde_json::from_value(activity).unwrap(); + let activity: Return = serde_json::from_value(activity).unwrap(); web::Json(activity) } -#[actix_rt::test] -async fn test_expired_signature() { +fn create_http_request() -> HttpRequest { let time1 = Utc::now().timestamp(); let time2 = Utc::now().timestamp(); let signature = format!( r#"keyId="my-key-id",algorithm="hs2019",created="{}",expires="{}",headers="(request-target) (created) (expires) date content-type",signature="blah blah blah""#, time1, time2 ); - let request = TestRequest::post() - .uri("http://localhost:8536/inbox") + TestRequest::post() + .uri("http://localhost:8536/") .header("Signature", signature) - .to_http_request(); + .to_http_request() +} + +#[actix_rt::test] +async fn test_shared_inbox_expired_signature() { + let request = create_http_request(); let context = create_context(); - let user = create_user(&context.pool().get().unwrap()); - let activity = create_activity(user.actor_id); + let user = create_user(&context.pool().get().unwrap(), "shared_inbox_rvgfd"); + let activity = + create_activity::>(user.actor_id); let response = shared_inbox(request, activity, web::Data::new(context)).await; assert_eq!( format!("{}", response.err().unwrap()), format!("{}", PrepareVerifyError::Expired) ); } + +#[actix_rt::test] +async fn test_user_inbox_expired_signature() { + let request = create_http_request(); + let context = create_context(); + let user = create_user(&context.pool().get().unwrap(), "user_inbox_cgsax"); + let activity = + create_activity::>(user.actor_id); + let path = Path:: { + 0: "username".to_string(), + }; + let response = user_inbox(request, activity, path, web::Data::new(context)).await; + assert_eq!( + format!("{}", response.err().unwrap()), + format!("{}", PrepareVerifyError::Expired) + ); +} + +#[actix_rt::test] +async fn test_community_inbox_expired_signature() { + let context = create_context(); + let user = create_user(&context.pool().get().unwrap(), "community_inbox_hrxa"); + let community = create_community(&context.pool().get().unwrap(), user.id); + let request = create_http_request(); + let activity = + create_activity::>(user.actor_id); + let path = Path:: { 0: community.name }; + let response = community_inbox(request, activity, path, web::Data::new(context)).await; + assert_eq!( + format!("{}", response.err().unwrap()), + format!("{}", PrepareVerifyError::Expired) + ); +} From 12af0f462f55a9c102daf832aa5936430f161346 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 21 Sep 2020 14:02:40 +0200 Subject: [PATCH 3/4] Update federation docs --- .travis.yml | 2 +- .../{run-tests.sh => run-tests.bash} | 0 docs/src/contributing_docker_development.md | 10 ++++++-- .../contributing_federation_development.md | 23 +------------------ docs/src/contributing_tests.md | 8 +++++-- 5 files changed, 16 insertions(+), 27 deletions(-) rename docker/federation/{run-tests.sh => run-tests.bash} (100%) diff --git a/.travis.yml b/.travis.yml index 350e0a8e8..648749043 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ before_install: # Change dir - cd docker/travis script: -- "./run-tests.sh" +- "./run-tests.bash" deploy: provider: script script: bash docker_push.sh diff --git a/docker/federation/run-tests.sh b/docker/federation/run-tests.bash similarity index 100% rename from docker/federation/run-tests.sh rename to docker/federation/run-tests.bash diff --git a/docs/src/contributing_docker_development.md b/docs/src/contributing_docker_development.md index 23b9fa005..586055a06 100644 --- a/docs/src/contributing_docker_development.md +++ b/docs/src/contributing_docker_development.md @@ -1,10 +1,16 @@ # Docker Development +## Dependencies (on Ubuntu) + +```bash +sudo apt install git docker-compose +sudo systemctl start docker +git clone https://github.com/LemmyNet/lemmy +``` + ## Running ```bash -sudo apt install git docker-compose -git clone https://github.com/LemmyNet/lemmy cd lemmy/docker/dev sudo docker-compose up --no-deps --build ``` diff --git a/docs/src/contributing_federation_development.md b/docs/src/contributing_federation_development.md index 08c4e5ca9..1d2436033 100644 --- a/docs/src/contributing_federation_development.md +++ b/docs/src/contributing_federation_development.md @@ -1,21 +1,9 @@ # Federation Development -## Setup - -If you don't have a local clone of the Lemmy repo yet, just run the following command: - -```bash -git clone https://github.com/LemmyNet/lemmy -``` - ## Running locally -You need to have the following packages installed, the Docker service needs to be running. +Install the dependencies as described in [Docker development](contributing_docker_development.md). Then run the following -- docker -- docker-compose - -Then run the following ```bash cd docker/federation ./start-local-instances.bash @@ -40,15 +28,6 @@ To start federation between instances, visit one of them and search for a user, Firefox containers are a good way to test them interacting. -## Integration tests - -To run a suite of suite of federation integration tests: - -```bash -cd docker/federation -./run-tests.bash -``` - ## Running on a server Note that federation is currently in alpha. **Only use it for testing**, not on any production server, and be aware that turning on federation may break your instance. diff --git a/docs/src/contributing_tests.md b/docs/src/contributing_tests.md index e40836c30..494cf5cd9 100644 --- a/docs/src/contributing_tests.md +++ b/docs/src/contributing_tests.md @@ -12,5 +12,9 @@ psql -U lemmy -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" ### Federation -Install the [Docker development dependencies](contributing_docker_development.md), and execute -`docker/federation-test/run-tests.sh` +Install the [Docker development dependencies](contributing_docker_development.md), and execute: + +``` +cd docker/federation +./run-tests.bash +``` From aece5e67b7c6c8432e7a05bd75d0238ee9795c72 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 21 Sep 2020 17:24:42 +0200 Subject: [PATCH 4/4] Address review comments --- docker/travis/{run-tests.sh => run-tests.bash} | 0 test.sh | 2 +- tests/integration_test.rs | 15 +++++++++++---- 3 files changed, 12 insertions(+), 5 deletions(-) rename docker/travis/{run-tests.sh => run-tests.bash} (100%) diff --git a/docker/travis/run-tests.sh b/docker/travis/run-tests.bash similarity index 100% rename from docker/travis/run-tests.sh rename to docker/travis/run-tests.bash diff --git a/test.sh b/test.sh index 801e95a41..beb499bdf 100755 --- a/test.sh +++ b/test.sh @@ -2,4 +2,4 @@ export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy diesel migration run export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy -RUST_TEST_THREADS=1 RUST_BACKTRACE=1 cargo test -j8 --no-fail-fast -- --nocapture +RUST_TEST_THREADS=1 cargo test --workspace --no-fail-fast diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 382bd65da..4f280f629 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -157,7 +157,8 @@ fn create_http_request() -> HttpRequest { async fn test_shared_inbox_expired_signature() { let request = create_http_request(); let context = create_context(); - let user = create_user(&context.pool().get().unwrap(), "shared_inbox_rvgfd"); + let connection = &context.pool().get().unwrap(); + let user = create_user(connection, "shared_inbox_rvgfd"); let activity = create_activity::>(user.actor_id); let response = shared_inbox(request, activity, web::Data::new(context)).await; @@ -165,13 +166,15 @@ async fn test_shared_inbox_expired_signature() { format!("{}", response.err().unwrap()), format!("{}", PrepareVerifyError::Expired) ); + User_::delete(connection, user.id).unwrap(); } #[actix_rt::test] async fn test_user_inbox_expired_signature() { let request = create_http_request(); let context = create_context(); - let user = create_user(&context.pool().get().unwrap(), "user_inbox_cgsax"); + let connection = &context.pool().get().unwrap(); + let user = create_user(connection, "user_inbox_cgsax"); let activity = create_activity::>(user.actor_id); let path = Path:: { @@ -182,13 +185,15 @@ async fn test_user_inbox_expired_signature() { format!("{}", response.err().unwrap()), format!("{}", PrepareVerifyError::Expired) ); + User_::delete(connection, user.id).unwrap(); } #[actix_rt::test] async fn test_community_inbox_expired_signature() { let context = create_context(); - let user = create_user(&context.pool().get().unwrap(), "community_inbox_hrxa"); - let community = create_community(&context.pool().get().unwrap(), user.id); + let connection = &context.pool().get().unwrap(); + let user = create_user(connection, "community_inbox_hrxa"); + let community = create_community(connection, user.id); let request = create_http_request(); let activity = create_activity::>(user.actor_id); @@ -198,4 +203,6 @@ async fn test_community_inbox_expired_signature() { format!("{}", response.err().unwrap()), format!("{}", PrepareVerifyError::Expired) ); + User_::delete(connection, user.id).unwrap(); + Community::delete(connection, community.id).unwrap(); }