lemmy/src/lib.rs

386 lines
13 KiB
Rust
Raw Normal View History

pub mod api_routes_http;
pub mod code_migrations;
pub mod prometheus_metrics;
pub mod root_span_builder;
pub mod scheduled_tasks;
pub mod session_middleware;
#[cfg(feature = "console")]
2022-05-10 12:06:32 +00:00
pub mod telemetry;
use crate::{
code_migrations::run_advanced_migrations,
root_span_builder::QuieterRootSpanBuilder,
session_middleware::SessionMiddleware,
};
use activitypub_federation::config::{FederationConfig, FederationMiddleware};
use actix_cors::Cors;
Fix #3366: Wrap plain-text error responses from the API in JSON (#3559) * Fix #3366: API does return plain HTML errors * Fix Clippy errors * Improve api response times by doing send_activity asynchronously (#3493) * do send_activity after http response * move to util function * format * fix prometheus * make synchronous federation configurable * cargo fmt * empty * empty --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> * Updating `login.rs` with generic `incorrect_login` response. (#3549) * Adding v0.18.1 and v0.18.0 release notes. (#3530) * Update RELEASES.md (#3556) added instruction to find the location of your docker directory (especially useful for those who used ansible since they never had to setup docker manually) * Use async email sender (#3554) * Upgrade all dependencies (#3526) * Upgrade all dependencies * as base64 * Adding phiresky to codeowners. (#3576) * Error enum fixed (#3487) * Create error type enum * Replace magic string slices with LemmyErrorTypes * Remove unused enum * Add rename snake case to error enum * Rename functions * clippy * Fix merge errors * Serialize in PascalCase instead of snake_case * Revert src/lib * Add serialization tests * Update translations * Fix compilation error in test * Fix another compilation error * Add code for generating typescript types * Various fixes to avoid breaking api * impl From<LemmyErrorType> for LemmyError * with_lemmy_type * trigger ci --------- Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> * Only update site_aggregates for local site (#3516) * Fix #3501 - Fix aggregation counts for elements removed and deleted (#3543) Two bugs were found and fixed: - previously elements removal and deletion were counted as two separate disappearances - removing comments did not affect post aggregations * Use LemmyErrorType also make error_type compulsory * Add missing import for jsonify_plain_text_errors * Fix formatting * Trying to make woodpecker run again --------- Co-authored-by: phiresky <phireskyde+git@gmail.com> Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: rosenjcb <rosenjcb@gmail.com> Co-authored-by: nixoye <12674582+nixoye@users.noreply.github.com> Co-authored-by: dullbananas <dull.bananas0@gmail.com> Co-authored-by: Nutomic <me@nutomic.com> Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> Co-authored-by: Sander Saarend <sander@saarend.com> Co-authored-by: Piotr Juszczyk <74842304+pijuszczyk@users.noreply.github.com>
2023-07-10 20:44:14 +00:00
use actix_web::{
dev::{ServerHandle, ServiceResponse},
middleware::{self, Condition, ErrorHandlerResponse, ErrorHandlers},
Fix #3366: Wrap plain-text error responses from the API in JSON (#3559) * Fix #3366: API does return plain HTML errors * Fix Clippy errors * Improve api response times by doing send_activity asynchronously (#3493) * do send_activity after http response * move to util function * format * fix prometheus * make synchronous federation configurable * cargo fmt * empty * empty --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> * Updating `login.rs` with generic `incorrect_login` response. (#3549) * Adding v0.18.1 and v0.18.0 release notes. (#3530) * Update RELEASES.md (#3556) added instruction to find the location of your docker directory (especially useful for those who used ansible since they never had to setup docker manually) * Use async email sender (#3554) * Upgrade all dependencies (#3526) * Upgrade all dependencies * as base64 * Adding phiresky to codeowners. (#3576) * Error enum fixed (#3487) * Create error type enum * Replace magic string slices with LemmyErrorTypes * Remove unused enum * Add rename snake case to error enum * Rename functions * clippy * Fix merge errors * Serialize in PascalCase instead of snake_case * Revert src/lib * Add serialization tests * Update translations * Fix compilation error in test * Fix another compilation error * Add code for generating typescript types * Various fixes to avoid breaking api * impl From<LemmyErrorType> for LemmyError * with_lemmy_type * trigger ci --------- Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> * Only update site_aggregates for local site (#3516) * Fix #3501 - Fix aggregation counts for elements removed and deleted (#3543) Two bugs were found and fixed: - previously elements removal and deletion were counted as two separate disappearances - removing comments did not affect post aggregations * Use LemmyErrorType also make error_type compulsory * Add missing import for jsonify_plain_text_errors * Fix formatting * Trying to make woodpecker run again --------- Co-authored-by: phiresky <phireskyde+git@gmail.com> Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: rosenjcb <rosenjcb@gmail.com> Co-authored-by: nixoye <12674582+nixoye@users.noreply.github.com> Co-authored-by: dullbananas <dull.bananas0@gmail.com> Co-authored-by: Nutomic <me@nutomic.com> Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> Co-authored-by: Sander Saarend <sander@saarend.com> Co-authored-by: Piotr Juszczyk <74842304+pijuszczyk@users.noreply.github.com>
2023-07-10 20:44:14 +00:00
web::Data,
App,
HttpResponse,
Fix #3366: Wrap plain-text error responses from the API in JSON (#3559) * Fix #3366: API does return plain HTML errors * Fix Clippy errors * Improve api response times by doing send_activity asynchronously (#3493) * do send_activity after http response * move to util function * format * fix prometheus * make synchronous federation configurable * cargo fmt * empty * empty --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> * Updating `login.rs` with generic `incorrect_login` response. (#3549) * Adding v0.18.1 and v0.18.0 release notes. (#3530) * Update RELEASES.md (#3556) added instruction to find the location of your docker directory (especially useful for those who used ansible since they never had to setup docker manually) * Use async email sender (#3554) * Upgrade all dependencies (#3526) * Upgrade all dependencies * as base64 * Adding phiresky to codeowners. (#3576) * Error enum fixed (#3487) * Create error type enum * Replace magic string slices with LemmyErrorTypes * Remove unused enum * Add rename snake case to error enum * Rename functions * clippy * Fix merge errors * Serialize in PascalCase instead of snake_case * Revert src/lib * Add serialization tests * Update translations * Fix compilation error in test * Fix another compilation error * Add code for generating typescript types * Various fixes to avoid breaking api * impl From<LemmyErrorType> for LemmyError * with_lemmy_type * trigger ci --------- Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> * Only update site_aggregates for local site (#3516) * Fix #3501 - Fix aggregation counts for elements removed and deleted (#3543) Two bugs were found and fixed: - previously elements removal and deletion were counted as two separate disappearances - removing comments did not affect post aggregations * Use LemmyErrorType also make error_type compulsory * Add missing import for jsonify_plain_text_errors * Fix formatting * Trying to make woodpecker run again --------- Co-authored-by: phiresky <phireskyde+git@gmail.com> Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: rosenjcb <rosenjcb@gmail.com> Co-authored-by: nixoye <12674582+nixoye@users.noreply.github.com> Co-authored-by: dullbananas <dull.bananas0@gmail.com> Co-authored-by: Nutomic <me@nutomic.com> Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> Co-authored-by: Sander Saarend <sander@saarend.com> Co-authored-by: Piotr Juszczyk <74842304+pijuszczyk@users.noreply.github.com>
2023-07-10 20:44:14 +00:00
HttpServer,
Result,
};
use actix_web_prom::PrometheusMetricsBuilder;
use clap::Parser;
use lemmy_api_common::{
context::LemmyContext,
lemmy_db_views::structs::SiteView,
request::client_builder,
send_activity::{ActivityChannel, MATCH_OUTGOING_ACTIVITIES},
utils::{
check_private_instance_and_federation_enabled,
local_site_rate_limit_to_rate_limit_config,
},
};
use lemmy_apub::{
activities::{handle_outgoing_activities, match_outgoing_activities},
objects::instance::ApubSite,
VerifyUrlData,
FEDERATION_HTTP_FETCH_LIMIT,
};
use lemmy_db_schema::{source::secret::Secret, utils::build_db_pool};
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
use lemmy_federate::{start_stop_federation_workers_cancellable, Opts};
use lemmy_routes::{feeds, images, nodeinfo, webfinger};
use lemmy_utils::{
error::LemmyError,
rate_limit::RateLimitCell,
Fix #3366: Wrap plain-text error responses from the API in JSON (#3559) * Fix #3366: API does return plain HTML errors * Fix Clippy errors * Improve api response times by doing send_activity asynchronously (#3493) * do send_activity after http response * move to util function * format * fix prometheus * make synchronous federation configurable * cargo fmt * empty * empty --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> * Updating `login.rs` with generic `incorrect_login` response. (#3549) * Adding v0.18.1 and v0.18.0 release notes. (#3530) * Update RELEASES.md (#3556) added instruction to find the location of your docker directory (especially useful for those who used ansible since they never had to setup docker manually) * Use async email sender (#3554) * Upgrade all dependencies (#3526) * Upgrade all dependencies * as base64 * Adding phiresky to codeowners. (#3576) * Error enum fixed (#3487) * Create error type enum * Replace magic string slices with LemmyErrorTypes * Remove unused enum * Add rename snake case to error enum * Rename functions * clippy * Fix merge errors * Serialize in PascalCase instead of snake_case * Revert src/lib * Add serialization tests * Update translations * Fix compilation error in test * Fix another compilation error * Add code for generating typescript types * Various fixes to avoid breaking api * impl From<LemmyErrorType> for LemmyError * with_lemmy_type * trigger ci --------- Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> * Only update site_aggregates for local site (#3516) * Fix #3501 - Fix aggregation counts for elements removed and deleted (#3543) Two bugs were found and fixed: - previously elements removal and deletion were counted as two separate disappearances - removing comments did not affect post aggregations * Use LemmyErrorType also make error_type compulsory * Add missing import for jsonify_plain_text_errors * Fix formatting * Trying to make woodpecker run again --------- Co-authored-by: phiresky <phireskyde+git@gmail.com> Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: rosenjcb <rosenjcb@gmail.com> Co-authored-by: nixoye <12674582+nixoye@users.noreply.github.com> Co-authored-by: dullbananas <dull.bananas0@gmail.com> Co-authored-by: Nutomic <me@nutomic.com> Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> Co-authored-by: Sander Saarend <sander@saarend.com> Co-authored-by: Piotr Juszczyk <74842304+pijuszczyk@users.noreply.github.com>
2023-07-10 20:44:14 +00:00
response::jsonify_plain_text_errors,
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
settings::{structs::Settings, SETTINGS},
VERSION,
};
use prometheus::default_registry;
use prometheus_metrics::serve_prometheus;
Rewrite images to use local proxy (#4035) * Add markdown rule to add rel=nofollow for all links * Add markdown image rule to add local image proxy (fixes #1036) * comments * rewrite markdown image links working * add comment * perform markdown image processing in api/apub receivers * clippy * add db table to validate proxied links * rewrite link fields for avatar, banner etc * sql fmt * proxy links received over federation * add config option * undo post.url rewriting, move http route definition * add tests * proxy images through pictrs * testing * cleanup request.rs file * more cleanup (fixes #2611) * include url content type when sending post over apub (fixes #2611) * store post url content type in db * should be media_type * get rid of cache_remote_thumbnails setting, instead automatically take thumbnail from federation data if available. * fix tests * add setting disable_external_link_previews * federate post url as image depending on mime type * change setting again * machete * invert * support custom emoji * clippy * update defaults * add image proxy test, fix test * fix test * clippy * revert accidental changes * address review * clippy * Markdown link rule-dess (#4356) * Extracting opengraph_data to its own type. * A few additions for markdown-link-rule. --------- Co-authored-by: Nutomic <me@nutomic.com> * fix setting * use enum for image proxy setting * fix test configs * add config backwards compat * clippy * machete --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com>
2024-01-25 14:22:11 +00:00
use reqwest_middleware::ClientBuilder;
use reqwest_tracing::TracingMiddleware;
use serde_json::json;
use std::{env, ops::Deref};
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
use tokio::signal::unix::SignalKind;
2021-11-23 12:16:47 +00:00
use tracing::subscriber::set_global_default;
use tracing_actix_web::TracingLogger;
2021-11-23 12:16:47 +00:00
use tracing_error::ErrorLayer;
use tracing_log::LogTracer;
use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry};
use url::Url;
2021-11-23 12:16:47 +00:00
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
#[derive(Parser, Debug)]
#[command(
version,
about = "A link aggregator for the fediverse",
long_about = "A link aggregator for the fediverse.\n\nThis is the Lemmy backend API server. This will connect to a PostgreSQL database, run any pending migrations and start accepting API requests."
)]
pub struct CmdArgs {
/// Don't run scheduled tasks.
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
///
/// If you are running multiple Lemmy server processes, you probably want to disable scheduled tasks on
/// all but one of the processes, to avoid running the tasks more often than intended.
#[arg(long, default_value_t = false)]
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
disable_scheduled_tasks: bool,
/// Disables the HTTP server.
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
///
/// This can be used to run a Lemmy server process that only performs scheduled tasks or activity sending.
#[arg(long, default_value_t = false)]
disable_http_server: bool,
/// Disable sending outgoing ActivityPub messages.
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
///
/// Only pass this for horizontally scaled setups.
/// See https://join-lemmy.org/docs/administration/horizontal_scaling.html for details.
#[arg(long, default_value_t = false)]
disable_activity_sending: bool,
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
/// The index of this outgoing federation process.
///
/// Defaults to 1/1. If you want to split the federation workload onto n servers, run each server 1≤i≤n with these args:
/// --federate-process-index i --federate-process-count n
///
/// Make you have exactly one server with each `i` running, otherwise federation will randomly send duplicates or nothing.
///
/// See https://join-lemmy.org/docs/administration/horizontal_scaling.html for more detail.
#[arg(long, default_value_t = 1)]
federate_process_index: i32,
/// How many outgoing federation processes you are starting in total.
///
/// If set, make sure to set --federate-process-index differently for each.
#[arg(long, default_value_t = 1)]
federate_process_count: i32,
}
/// Placing the main function in lib.rs allows other crates to import it and embed Lemmy
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> {
// Print version number to log
println!("Lemmy v{VERSION}");
// return error 503 while running db migrations and startup tasks
let mut startup_server_handle = None;
if !args.disable_http_server {
startup_server_handle = Some(create_startup_server()?);
}
// Set up the connection pool
let pool = build_db_pool().await?;
// Run the Code-required migrations
run_advanced_migrations(&mut (&pool).into(), &SETTINGS).await?;
// Initialize the secrets
Make functions work with both connection and pool (#3420) * a lot * merge * Fix stuff broken by merge * Get rid of repetitive `&mut *context.conn().await?` * Add blank lines under each line with `conn =` * Fix style mistakes (partial) * Revert "Fix style mistakes (partial)" This reverts commit 48a033b87f4fdc1ce14ff86cc019e1c703cd2741. * Revert "Add blank lines under each line with `conn =`" This reverts commit 773a6d3beba2cf89eac75913078b40c4f5190dd4. * Revert "Get rid of repetitive `&mut *context.conn().await?`" This reverts commit d2c6263ea13710177d49b2791278db5ad115fca5. * Use DbConn for CaptchaAnswer methods * DbConn trait * Remove more `&mut *` * Fix stuff * Re-run CI * try to make ci start * fix * fix * Fix api_common::utils * Fix apub::activities::block * Fix apub::api::resolve_object * Fix some things * Revert "Fix some things" This reverts commit 2bf8574bc8333d8d34ca542d61a0a5b50039c24d. * Revert "Fix apub::api::resolve_object" This reverts commit 3e4059aabbe485b2ff060bdeced8ef958ff62832. * Revert "Fix apub::activities::block" This reverts commit 3b02389abd780a7b1b8a2c89e26febdaa6a12159. * Revert "Fix api_common::utils" This reverts commit 7dc73de613a5618fa57eb06450f3699bbcb41254. * Revert "Revert "Fix api_common::utils"" This reverts commit f740f115e5457e83e53cc223e48196a2c47a9975. * Revert "Revert "Fix apub::activities::block"" This reverts commit 2ee206af7c885c10092cf209bf4a5b1d60327866. * Revert "Revert "Fix apub::api::resolve_object"" This reverts commit 96ed8bf2e9dcadae760743929498312334e23d2e. * Fix fetch_local_site_data * Fix get_comment_parent_creator * Remove unused perma deleted text * Fix routes::feeds * Fix lib.rs * Update lib.rs * rerun ci * Attempt to create custom GetConn and RunQueryDsl traits * Start over * Add GetConn trait * aaaa * Revert "aaaa" This reverts commit acc9ca1aed10c39efdd91cefece066e035a1fe80. * Revert "Revert "aaaa"" This reverts commit 443a2a00a56d152bb7eb429efd0d29a78e21b163. * still aaaaaaaaaaaaa * Return to earlier thing Revert "Add GetConn trait" This reverts commit ab4e94aea5bd9d34cbcddf017339131047e75344. * Try to use DbPool enum * Revert "Try to use DbPool enum" This reverts commit e4d1712646a52006b865a1fbe0dcf79976fdb027. * DbConn and DbPool enums (db_schema only fails to compile for tests) * fmt * Make functions take `&mut DbPool<'_>` and make db_schema tests compile * Add try_join_with_pool macro and run fix-clippy on more crates * Fix some errors * I did it * Remove function variants that take connection * rerun ci * rerun ci * rerun ci
2023-07-11 13:09:59 +00:00
let secret = Secret::init(&mut (&pool).into())
.await
.expect("Couldn't initialize secrets.");
// Make sure the local site is set up.
Make functions work with both connection and pool (#3420) * a lot * merge * Fix stuff broken by merge * Get rid of repetitive `&mut *context.conn().await?` * Add blank lines under each line with `conn =` * Fix style mistakes (partial) * Revert "Fix style mistakes (partial)" This reverts commit 48a033b87f4fdc1ce14ff86cc019e1c703cd2741. * Revert "Add blank lines under each line with `conn =`" This reverts commit 773a6d3beba2cf89eac75913078b40c4f5190dd4. * Revert "Get rid of repetitive `&mut *context.conn().await?`" This reverts commit d2c6263ea13710177d49b2791278db5ad115fca5. * Use DbConn for CaptchaAnswer methods * DbConn trait * Remove more `&mut *` * Fix stuff * Re-run CI * try to make ci start * fix * fix * Fix api_common::utils * Fix apub::activities::block * Fix apub::api::resolve_object * Fix some things * Revert "Fix some things" This reverts commit 2bf8574bc8333d8d34ca542d61a0a5b50039c24d. * Revert "Fix apub::api::resolve_object" This reverts commit 3e4059aabbe485b2ff060bdeced8ef958ff62832. * Revert "Fix apub::activities::block" This reverts commit 3b02389abd780a7b1b8a2c89e26febdaa6a12159. * Revert "Fix api_common::utils" This reverts commit 7dc73de613a5618fa57eb06450f3699bbcb41254. * Revert "Revert "Fix api_common::utils"" This reverts commit f740f115e5457e83e53cc223e48196a2c47a9975. * Revert "Revert "Fix apub::activities::block"" This reverts commit 2ee206af7c885c10092cf209bf4a5b1d60327866. * Revert "Revert "Fix apub::api::resolve_object"" This reverts commit 96ed8bf2e9dcadae760743929498312334e23d2e. * Fix fetch_local_site_data * Fix get_comment_parent_creator * Remove unused perma deleted text * Fix routes::feeds * Fix lib.rs * Update lib.rs * rerun ci * Attempt to create custom GetConn and RunQueryDsl traits * Start over * Add GetConn trait * aaaa * Revert "aaaa" This reverts commit acc9ca1aed10c39efdd91cefece066e035a1fe80. * Revert "Revert "aaaa"" This reverts commit 443a2a00a56d152bb7eb429efd0d29a78e21b163. * still aaaaaaaaaaaaa * Return to earlier thing Revert "Add GetConn trait" This reverts commit ab4e94aea5bd9d34cbcddf017339131047e75344. * Try to use DbPool enum * Revert "Try to use DbPool enum" This reverts commit e4d1712646a52006b865a1fbe0dcf79976fdb027. * DbConn and DbPool enums (db_schema only fails to compile for tests) * fmt * Make functions take `&mut DbPool<'_>` and make db_schema tests compile * Add try_join_with_pool macro and run fix-clippy on more crates * Fix some errors * I did it * Remove function variants that take connection * rerun ci * rerun ci * rerun ci
2023-07-11 13:09:59 +00:00
let site_view = SiteView::read_local(&mut (&pool).into())
.await
.expect("local site not set up");
let local_site = site_view.local_site;
let federation_enabled = local_site.federation_enabled;
if federation_enabled {
println!("Federation enabled, host is {}", &SETTINGS.hostname);
}
check_private_instance_and_federation_enabled(&local_site)?;
// Set up the rate limiter
let rate_limit_config =
local_site_rate_limit_to_rate_limit_config(&site_view.local_site_rate_limit);
Refactor rate limiter and improve rate limit bucket cleanup (#3937) * Update rate_limiter.rs * Update mod.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update mod.rs * Update scheduled_tasks.rs * Shrink `RateLimitBucket` * Update rate_limiter.rs * Update mod.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update mod.rs * Update rate_limiter.rs * fmt * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * rerun ci * Update rate_limiter.rs * Undo changes to fields * Manually undo changes to RateLimitBucket fields * fmt * Bucket cleanup loop in rate_limit/mod.rs * Remove rate limit bucket cleanup from scheduled_tasks.rs * Remove ; * Remove UNINITIALIZED_TOKEN_AMOUNT * Update rate_limiter.rs * fmt * Update rate_limiter.rs * fmt * Update rate_limiter.rs * fmt * Update rate_limiter.rs * stuff * MapLevel trait * fix merge * Prevent negative numbers in buckets * Clean up MapLevel::check * MapLevel::remove_full_buckets * stuff * Use remove_full_buckets to avoid allocations * stuff * remove tx * Remove RateLimitConfig * Rename settings_updated_channel to rate_limit_cell * Remove global rate limit cell * impl Default for RateLimitCell * bucket_configs doc comment to explain EnumMap * improve test_rate_limiter * rename default to with_test_config --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: Nutomic <me@nutomic.com>
2023-10-19 13:31:51 +00:00
let rate_limit_cell = RateLimitCell::new(rate_limit_config);
println!(
"Starting HTTP server at {}:{}",
SETTINGS.bind, SETTINGS.port
);
let client = ClientBuilder::new(client_builder(&SETTINGS).build()?)
.with(TracingMiddleware::default())
.build();
let context = LemmyContext::create(
pool.clone(),
client.clone(),
secret.clone(),
rate_limit_cell.clone(),
);
if !args.disable_scheduled_tasks {
// Schedules various cleanup tasks for the DB
let _scheduled_tasks = tokio::task::spawn(scheduled_tasks::setup(context.clone()));
}
if let Some(prometheus) = SETTINGS.prometheus.clone() {
serve_prometheus(prometheus, context.clone())?;
}
let mut federation_config = FederationConfig::builder();
federation_config
.domain(SETTINGS.hostname.clone())
.app_data(context.clone())
.client(client.clone())
.http_fetch_limit(FEDERATION_HTTP_FETCH_LIMIT)
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
.debug(cfg!(debug_assertions))
.http_signature_compat(true)
.url_verifier(Box::new(VerifyUrlData(context.inner_pool().clone())));
if local_site.federation_signed_fetch {
let site: ApubSite = site_view.site.into();
federation_config.signed_fetch_actor(&site);
}
let federation_config = federation_config.build().await?;
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
MATCH_OUTGOING_ACTIVITIES
.set(Box::new(move |d, c| {
Box::pin(match_outgoing_activities(d, c))
}))
.expect("set function pointer");
let request_data = federation_config.to_request_data();
let outgoing_activities_task = tokio::task::spawn(handle_outgoing_activities(request_data));
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
let server = if !args.disable_http_server {
if let Some(startup_server_handle) = startup_server_handle {
startup_server_handle.stop(true).await;
}
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
Some(create_http_server(
federation_config.clone(),
SETTINGS.clone(),
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
federation_enabled,
)?)
} else {
None
};
let federate = (!args.disable_activity_sending).then(|| {
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
start_stop_federation_workers_cancellable(
Opts {
process_index: args.federate_process_index,
process_count: args.federate_process_count,
},
pool.clone(),
federation_config.clone(),
)
});
let mut interrupt = tokio::signal::unix::signal(SignalKind::interrupt())?;
let mut terminate = tokio::signal::unix::signal(SignalKind::terminate())?;
tokio::select! {
_ = tokio::signal::ctrl_c() => {
tracing::warn!("Received ctrl-c, shutting down gracefully...");
}
_ = interrupt.recv() => {
tracing::warn!("Received interrupt, shutting down gracefully...");
}
_ = terminate.recv() => {
tracing::warn!("Received terminate, shutting down gracefully...");
}
}
if let Some(server) = server {
server.stop(true).await;
}
if let Some(federate) = federate {
federate.cancel().await?;
}
// Wait for outgoing apub sends to complete
ActivityChannel::close(outgoing_activities_task).await?;
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
Ok(())
}
/// Creates temporary HTTP server which returns status 503 for all requests.
fn create_startup_server() -> Result<ServerHandle, LemmyError> {
let startup_server = HttpServer::new(move || {
App::new().wrap(ErrorHandlers::new().default_handler(move |req| {
let (req, _) = req.into_parts();
let response =
HttpResponse::ServiceUnavailable().json(json!({"error": "Lemmy is currently starting"}));
let service_response = ServiceResponse::new(req, response);
Ok(ErrorHandlerResponse::Response(
service_response.map_into_right_body(),
))
}))
})
.bind((SETTINGS.bind, SETTINGS.port))?
.run();
let startup_server_handle = startup_server.handle();
tokio::task::spawn(startup_server);
Ok(startup_server_handle)
}
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
fn create_http_server(
federation_config: FederationConfig<LemmyContext>,
settings: Settings,
federation_enabled: bool,
) -> Result<ServerHandle, LemmyError> {
// this must come before the HttpServer creation
// creates a middleware that populates http metrics for each path, method, and status code
let prom_api_metrics = PrometheusMetricsBuilder::new("lemmy_api")
.registry(default_registry().clone())
.build()
.expect("Should always be buildable");
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
let context: LemmyContext = federation_config.deref().clone();
Refactor rate limiter and improve rate limit bucket cleanup (#3937) * Update rate_limiter.rs * Update mod.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update mod.rs * Update scheduled_tasks.rs * Shrink `RateLimitBucket` * Update rate_limiter.rs * Update mod.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update mod.rs * Update rate_limiter.rs * fmt * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * Update rate_limiter.rs * rerun ci * Update rate_limiter.rs * Undo changes to fields * Manually undo changes to RateLimitBucket fields * fmt * Bucket cleanup loop in rate_limit/mod.rs * Remove rate limit bucket cleanup from scheduled_tasks.rs * Remove ; * Remove UNINITIALIZED_TOKEN_AMOUNT * Update rate_limiter.rs * fmt * Update rate_limiter.rs * fmt * Update rate_limiter.rs * fmt * Update rate_limiter.rs * stuff * MapLevel trait * fix merge * Prevent negative numbers in buckets * Clean up MapLevel::check * MapLevel::remove_full_buckets * stuff * Use remove_full_buckets to avoid allocations * stuff * remove tx * Remove RateLimitConfig * Rename settings_updated_channel to rate_limit_cell * Remove global rate limit cell * impl Default for RateLimitCell * bucket_configs doc comment to explain EnumMap * improve test_rate_limiter * rename default to with_test_config --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: Nutomic <me@nutomic.com>
2023-10-19 13:31:51 +00:00
let rate_limit_cell = federation_config.rate_limit_cell().clone();
Rewrite images to use local proxy (#4035) * Add markdown rule to add rel=nofollow for all links * Add markdown image rule to add local image proxy (fixes #1036) * comments * rewrite markdown image links working * add comment * perform markdown image processing in api/apub receivers * clippy * add db table to validate proxied links * rewrite link fields for avatar, banner etc * sql fmt * proxy links received over federation * add config option * undo post.url rewriting, move http route definition * add tests * proxy images through pictrs * testing * cleanup request.rs file * more cleanup (fixes #2611) * include url content type when sending post over apub (fixes #2611) * store post url content type in db * should be media_type * get rid of cache_remote_thumbnails setting, instead automatically take thumbnail from federation data if available. * fix tests * add setting disable_external_link_previews * federate post url as image depending on mime type * change setting again * machete * invert * support custom emoji * clippy * update defaults * add image proxy test, fix test * fix test * clippy * revert accidental changes * address review * clippy * Markdown link rule-dess (#4356) * Extracting opengraph_data to its own type. * A few additions for markdown-link-rule. --------- Co-authored-by: Nutomic <me@nutomic.com> * fix setting * use enum for image proxy setting * fix test configs * add config backwards compat * clippy * machete --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com>
2024-01-25 14:22:11 +00:00
// Pictrs cannot use proxy
let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?)
.with(TracingMiddleware::default())
.build();
// Create Http server
let bind = (settings.bind, settings.port);
let server = HttpServer::new(move || {
let cors_config = cors_config(&settings);
let app = App::new()
.wrap(middleware::Logger::new(
// This is the default log format save for the usage of %{r}a over %a to guarantee to record the client's (forwarded) IP and not the last peer address, since the latter is frequently just a reverse proxy
"%{r}a '%r' %s %b '%{Referer}i' '%{User-Agent}i' %T",
))
2023-06-26 10:54:41 +00:00
.wrap(middleware::Compress::default())
.wrap(cors_config)
.wrap(TracingLogger::<QuieterRootSpanBuilder>::new())
Fix #3366: Wrap plain-text error responses from the API in JSON (#3559) * Fix #3366: API does return plain HTML errors * Fix Clippy errors * Improve api response times by doing send_activity asynchronously (#3493) * do send_activity after http response * move to util function * format * fix prometheus * make synchronous federation configurable * cargo fmt * empty * empty --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> * Updating `login.rs` with generic `incorrect_login` response. (#3549) * Adding v0.18.1 and v0.18.0 release notes. (#3530) * Update RELEASES.md (#3556) added instruction to find the location of your docker directory (especially useful for those who used ansible since they never had to setup docker manually) * Use async email sender (#3554) * Upgrade all dependencies (#3526) * Upgrade all dependencies * as base64 * Adding phiresky to codeowners. (#3576) * Error enum fixed (#3487) * Create error type enum * Replace magic string slices with LemmyErrorTypes * Remove unused enum * Add rename snake case to error enum * Rename functions * clippy * Fix merge errors * Serialize in PascalCase instead of snake_case * Revert src/lib * Add serialization tests * Update translations * Fix compilation error in test * Fix another compilation error * Add code for generating typescript types * Various fixes to avoid breaking api * impl From<LemmyErrorType> for LemmyError * with_lemmy_type * trigger ci --------- Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> * Only update site_aggregates for local site (#3516) * Fix #3501 - Fix aggregation counts for elements removed and deleted (#3543) Two bugs were found and fixed: - previously elements removal and deletion were counted as two separate disappearances - removing comments did not affect post aggregations * Use LemmyErrorType also make error_type compulsory * Add missing import for jsonify_plain_text_errors * Fix formatting * Trying to make woodpecker run again --------- Co-authored-by: phiresky <phireskyde+git@gmail.com> Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: rosenjcb <rosenjcb@gmail.com> Co-authored-by: nixoye <12674582+nixoye@users.noreply.github.com> Co-authored-by: dullbananas <dull.bananas0@gmail.com> Co-authored-by: Nutomic <me@nutomic.com> Co-authored-by: SleeplessOne1917 <abias1122@gmail.com> Co-authored-by: Sander Saarend <sander@saarend.com> Co-authored-by: Piotr Juszczyk <74842304+pijuszczyk@users.noreply.github.com>
2023-07-10 20:44:14 +00:00
.wrap(ErrorHandlers::new().default_handler(jsonify_plain_text_errors))
.app_data(Data::new(context.clone()))
.app_data(Data::new(rate_limit_cell.clone()))
.wrap(FederationMiddleware::new(federation_config.clone()))
.wrap(SessionMiddleware::new(context.clone()))
.wrap(Condition::new(
SETTINGS.prometheus.is_some(),
prom_api_metrics.clone(),
));
// The routes
app
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
.configure(|cfg| api_routes_http::config(cfg, &rate_limit_cell))
.configure(|cfg| {
if federation_enabled {
lemmy_apub::http::routes::config(cfg);
webfinger::config(cfg);
}
})
.configure(feeds::config)
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
.configure(|cfg| images::config(cfg, pictrs_client.clone(), &rate_limit_cell))
.configure(nodeinfo::config)
})
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
.disable_signals()
.bind(bind)?
Persistent, performant, reliable federation queue (#3605) * persistent activity queue * fixes * fixes * make federation workers function callable from outside * log federation instances * dead instance detection not needed here * taplo fmt * split federate bin/lib * minor fix * better logging * log * create struct to hold cancellable task for readability * use boxfuture for readability * reset submodule * fix * fix lint * swap * remove json column, use separate array columns instead * some review comments * make worker a struct for readability * minor readability * add local filter to community follower view * remove separate lemmy_federate entry point * fix remaining duration * address review comments mostly * fix lint * upgrade actitypub-fed to simpler interface * fix sql format * increase delays a bit * fixes after merge * remove selectable * fix instance selectable * add comment * start federation based on latest id at the time * rename federate process args * dead instances in one query * filter follow+report activities by local * remove synchronous federation remove activity sender queue * lint * fix federation tests by waiting for results to change * fix fed test * fix comment report * wait some more * Apply suggestions from code review Co-authored-by: SorteKanin <sortekanin@gmail.com> * fix most remaining tests * wait until private messages * fix community tests * fix community tests * move arg parse * use instance_id instead of domain in federation_queue_state table --------- Co-authored-by: Dessalines <dessalines@users.noreply.github.com> Co-authored-by: SorteKanin <sortekanin@gmail.com>
2023-09-09 16:25:03 +00:00
.run();
let handle = server.handle();
tokio::task::spawn(server);
Ok(handle)
}
fn cors_config(settings: &Settings) -> Cors {
let self_origin = settings.get_protocol_and_hostname();
let cors_origin_setting = settings.cors_origin();
match (cors_origin_setting.clone(), cfg!(debug_assertions)) {
(Some(origin), false) => {
// Need to call send_wildcard() explicitly, passing this into allowed_origin() results in error
if cors_origin_setting.as_deref() == Some("*") {
Cors::default().allow_any_origin().send_wildcard()
} else {
Cors::default()
.allowed_origin(&origin)
.allowed_origin(&self_origin)
}
}
_ => Cors::default()
.allow_any_origin()
.allow_any_method()
.allow_any_header()
.expose_any_header()
.max_age(3600),
}
}
pub fn init_logging(opentelemetry_url: &Option<Url>) -> Result<(), LemmyError> {
2021-11-23 12:16:47 +00:00
LogTracer::init()?;
let log_description = env::var("RUST_LOG").unwrap_or_else(|_| "info".into());
let targets = log_description
.trim()
.trim_matches('"')
.parse::<Targets>()?;
let format_layer = {
#[cfg(feature = "json-log")]
let layer = tracing_subscriber::fmt::layer().with_ansi(false).json();
#[cfg(not(feature = "json-log"))]
let layer = tracing_subscriber::fmt::layer().with_ansi(false);
layer.with_filter(targets.clone())
};
2021-11-23 12:16:47 +00:00
let subscriber = Registry::default()
.with(format_layer)
.with(ErrorLayer::default());
2022-05-10 12:06:32 +00:00
if let Some(_url) = opentelemetry_url {
#[cfg(feature = "console")]
telemetry::init_tracing(_url.as_ref(), subscriber, targets)?;
2022-05-10 12:06:32 +00:00
#[cfg(not(feature = "console"))]
tracing::error!("Feature `console` must be enabled for opentelemetry tracing");
} else {
set_global_default(subscriber)?;
}
2021-11-23 12:16:47 +00:00
Ok(())
}