Adding instance software and version. Fixes #2222 (#2733)

* Adding instance software and version. Fixes #2222

* Fix clippy.

* Fix clippy 2

* Fixing some more issues.
This commit is contained in:
Dessalines 2023-02-18 09:36:12 -05:00 committed by GitHub
parent 47f4aa3550
commit 3735c6fabf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 201 additions and 92 deletions

View file

@ -71,7 +71,7 @@ tracing-error = "0.2.0"
tracing-log = "0.1.3"
tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
url = { version = "2.3.1", features = ["serde"] }
reqwest = { version = "0.11.12", features = ["json"] }
reqwest = { version = "0.11.12", features = ["json", "blocking"] }
reqwest-middleware = "0.2.0"
reqwest-tracing = "0.4.0"
clokwerk = "0.3.5"

View file

@ -1,7 +1,12 @@
use crate::sensitive::Sensitive;
use lemmy_db_schema::{
newtypes::{CommentId, CommunityId, LanguageId, PersonId, PostId},
source::{language::Language, local_site::RegistrationMode, tagline::Tagline},
source::{
instance::Instance,
language::Language,
local_site::RegistrationMode,
tagline::Tagline,
},
ListingType,
ModlogActionType,
SearchType,
@ -239,9 +244,9 @@ pub struct LeaveAdmin {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct FederatedInstances {
pub linked: Vec<String>,
pub allowed: Option<Vec<String>>,
pub blocked: Option<Vec<String>>,
pub linked: Vec<Instance>,
pub allowed: Option<Vec<Instance>>,
pub blocked: Option<Vec<Instance>>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]

View file

@ -114,13 +114,13 @@ fn check_apub_id_valid(
}
if let Some(blocked) = local_site_data.blocked_instances.as_ref() {
if blocked.contains(&domain) {
if blocked.iter().any(|i| domain.eq(&i.domain)) {
return Err("Domain is blocked");
}
}
if let Some(allowed) = local_site_data.allowed_instances.as_ref() {
if !allowed.contains(&domain) {
if !allowed.iter().any(|i| domain.eq(&i.domain)) {
return Err("Domain is not in allowlist");
}
}
@ -131,8 +131,8 @@ fn check_apub_id_valid(
#[derive(Clone)]
pub(crate) struct LocalSiteData {
local_site: Option<LocalSite>,
allowed_instances: Option<Vec<String>>,
blocked_instances: Option<Vec<String>>,
allowed_instances: Option<Vec<Instance>>,
blocked_instances: Option<Vec<Instance>>,
}
pub(crate) async fn fetch_local_site_data(
@ -175,7 +175,10 @@ pub(crate) fn check_apub_id_valid_with_strictness(
if is_strict {
// need to allow this explicitly because apub receive might contain objects from our local
// instance.
let mut allowed_and_local = allowed.clone();
let mut allowed_and_local = allowed
.iter()
.map(|i| i.domain.clone())
.collect::<Vec<String>>();
allowed_and_local.push(local_instance);
if !allowed_and_local.contains(&domain) {

View file

@ -59,25 +59,24 @@ mod tests {
#[serial]
async fn test_allowlist_insert_and_clear() {
let pool = &build_db_pool_for_tests().await;
let allowed = Some(vec![
let domains = vec![
"tld1.xyz".to_string(),
"tld2.xyz".to_string(),
"tld3.xyz".to_string(),
]);
];
let allowed = Some(domains.clone());
FederationAllowList::replace(pool, allowed).await.unwrap();
let allows = Instance::allowlist(pool).await.unwrap();
let allows_domains = allows
.iter()
.map(|i| i.domain.clone())
.collect::<Vec<String>>();
assert_eq!(3, allows.len());
assert_eq!(
vec![
"tld1.xyz".to_string(),
"tld2.xyz".to_string(),
"tld3.xyz".to_string()
],
allows
);
assert_eq!(domains, allows_domains);
// Now test clearing them via Some(empty vec)
let clear_allows = Some(Vec::new());

View file

@ -31,10 +31,10 @@ impl Instance {
Self::create(pool, domain).await
}
pub async fn create_conn(conn: &mut AsyncPgConnection, domain: &str) -> Result<Self, Error> {
let form = InstanceForm {
domain: domain.to_string(),
updated: Some(naive_now()),
};
let form = InstanceForm::builder()
.domain(domain.to_string())
.updated(Some(naive_now()))
.build();
Self::create_from_form_conn(conn, &form).await
}
pub async fn delete(pool: &DbPool, instance_id: InstanceId) -> Result<usize, Error> {
@ -47,31 +47,31 @@ impl Instance {
let conn = &mut get_conn(pool).await?;
diesel::delete(instance::table).execute(conn).await
}
pub async fn allowlist(pool: &DbPool) -> Result<Vec<String>, Error> {
pub async fn allowlist(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
instance::table
.inner_join(federation_allowlist::table)
.select(instance::domain)
.load::<String>(conn)
.select(instance::all_columns)
.get_results(conn)
.await
}
pub async fn blocklist(pool: &DbPool) -> Result<Vec<String>, Error> {
pub async fn blocklist(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
instance::table
.inner_join(federation_blocklist::table)
.select(instance::domain)
.load::<String>(conn)
.select(instance::all_columns)
.get_results(conn)
.await
}
pub async fn linked(pool: &DbPool) -> Result<Vec<String>, Error> {
pub async fn linked(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
instance::table
.left_join(federation_blocklist::table)
.filter(federation_blocklist::id.is_null())
.select(instance::domain)
.load::<String>(conn)
.select(instance::all_columns)
.get_results(conn)
.await
}
}

View file

@ -648,6 +648,8 @@ table! {
instance(id) {
id -> Int4,
domain -> Text,
software -> Nullable<Text>,
version -> Nullable<Text>,
published -> Timestamp,
updated -> Nullable<Timestamp>,
}

View file

@ -1,21 +1,30 @@
use crate::newtypes::InstanceId;
#[cfg(feature = "full")]
use crate::schema::instance;
use serde::{Deserialize, Serialize};
use std::fmt::Debug;
use typed_builder::TypedBuilder;
#[derive(PartialEq, Eq, Debug)]
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = instance))]
pub struct Instance {
pub id: InstanceId,
pub domain: String,
pub software: Option<String>,
pub version: Option<String>,
pub published: chrono::NaiveDateTime,
pub updated: Option<chrono::NaiveDateTime>,
}
#[derive(Clone, TypedBuilder)]
#[builder(field_defaults(default))]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = instance))]
pub struct InstanceForm {
#[builder(!default)]
pub domain: String,
pub software: Option<String>,
pub version: Option<String>,
pub updated: Option<chrono::NaiveDateTime>,
}

View file

@ -34,27 +34,27 @@ async fn node_info(context: web::Data<LemmyContext>) -> Result<HttpResponse, Err
.map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?;
let protocols = if site_view.local_site.federation_enabled {
vec!["activitypub".to_string()]
Some(vec!["activitypub".to_string()])
} else {
vec![]
None
};
let open_registrations = site_view.local_site.registration_mode == RegistrationMode::Open;
let open_registrations = Some(site_view.local_site.registration_mode == RegistrationMode::Open);
let json = NodeInfo {
version: "2.0".to_string(),
software: NodeInfoSoftware {
name: "lemmy".to_string(),
version: version::VERSION.to_string(),
},
version: Some("2.0".to_string()),
software: Some(NodeInfoSoftware {
name: Some("lemmy".to_string()),
version: Some(version::VERSION.to_string()),
}),
protocols,
usage: NodeInfoUsage {
users: NodeInfoUsers {
total: site_view.counts.users,
active_halfyear: site_view.counts.users_active_half_year,
active_month: site_view.counts.users_active_month,
},
local_posts: site_view.counts.posts,
local_comments: site_view.counts.comments,
},
usage: Some(NodeInfoUsage {
users: Some(NodeInfoUsers {
total: Some(site_view.counts.users),
active_halfyear: Some(site_view.counts.users_active_half_year),
active_month: Some(site_view.counts.users_active_month),
}),
local_posts: Some(site_view.counts.posts),
local_comments: Some(site_view.counts.comments),
}),
open_registrations,
};
@ -72,34 +72,35 @@ struct NodeInfoWellKnownLinks {
pub href: Url,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct NodeInfo {
pub version: String,
pub software: NodeInfoSoftware,
pub protocols: Vec<String>,
pub usage: NodeInfoUsage,
pub open_registrations: bool,
#[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase", default)]
pub struct NodeInfo {
pub version: Option<String>,
pub software: Option<NodeInfoSoftware>,
pub protocols: Option<Vec<String>>,
pub usage: Option<NodeInfoUsage>,
pub open_registrations: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug)]
struct NodeInfoSoftware {
pub name: String,
pub version: String,
#[derive(Serialize, Deserialize, Debug, Default)]
#[serde(default)]
pub struct NodeInfoSoftware {
pub name: Option<String>,
pub version: Option<String>,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct NodeInfoUsage {
pub users: NodeInfoUsers,
pub local_posts: i64,
pub local_comments: i64,
#[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase", default)]
pub struct NodeInfoUsage {
pub users: Option<NodeInfoUsers>,
pub local_posts: Option<i64>,
pub local_comments: Option<i64>,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct NodeInfoUsers {
pub total: i64,
pub active_halfyear: i64,
pub active_month: i64,
#[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase", default)]
pub struct NodeInfoUsers {
pub total: Option<i64>,
pub active_halfyear: Option<i64>,
pub active_month: Option<i64>,
}

View file

@ -0,0 +1,2 @@
alter table instance drop column software;
alter table instance drop column version;

View file

@ -0,0 +1,4 @@
-- Add Software and Version columns from nodeinfo to the instance table
alter table instance add column software varchar(255);
alter table instance add column version varchar(255);

View file

@ -1,6 +1,10 @@
#!/bin/bash
set -e
CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
cd $CWD/../
cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-targets --all-features -- \
-D warnings -D deprecated -D clippy::perf -D clippy::complexity \
-D clippy::style -D clippy::correctness -D clippy::suspicious \
@ -10,3 +14,5 @@ cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-target
-D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \
-D clippy::unused_self \
-A clippy::uninlined_format_args
cargo +nightly fmt

View file

@ -42,7 +42,7 @@ use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry}
use url::Url;
/// Max timeout for http requests
const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
/// Placing the main function in lib.rs allows other crates to import it and embed Lemmy
pub async fn start_lemmy_server() -> Result<(), LemmyError> {
@ -73,11 +73,6 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
let pool = build_db_pool(&settings).await?;
run_advanced_migrations(&pool, &settings).await?;
// Schedules various cleanup tasks for the DB
thread::spawn(move || {
scheduled_tasks::setup(db_url).expect("Couldn't set up scheduled_tasks");
});
// Initialize the secrets
let secret = Secret::init(&pool)
.await
@ -106,8 +101,9 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
settings.bind, settings.port
);
let user_agent = build_user_agent(&settings);
let reqwest_client = Client::builder()
.user_agent(build_user_agent(&settings))
.user_agent(user_agent.clone())
.timeout(REQWEST_TIMEOUT)
.build()?;
@ -128,6 +124,11 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
.with(TracingMiddleware::default())
.build();
// Schedules various cleanup tasks for the DB
thread::spawn(move || {
scheduled_tasks::setup(db_url, user_agent).expect("Couldn't set up scheduled_tasks");
});
let chat_server = Arc::new(ChatServer::startup());
// Create Http server with websocket support

View file

@ -2,17 +2,25 @@ use clokwerk::{Scheduler, TimeUnits};
// Import week days and WeekDay
use diesel::{sql_query, PgConnection, RunQueryDsl};
use diesel::{Connection, ExpressionMethods, QueryDsl};
use lemmy_utils::error::LemmyError;
use lemmy_db_schema::{
source::instance::{Instance, InstanceForm},
utils::naive_now,
};
use lemmy_routes::nodeinfo::NodeInfo;
use lemmy_utils::{error::LemmyError, REQWEST_TIMEOUT};
use reqwest::blocking::Client;
use std::{thread, time::Duration};
use tracing::info;
/// Schedules various cleanup tasks for lemmy in a background thread
pub fn setup(db_url: String) -> Result<(), LemmyError> {
pub fn setup(db_url: String, user_agent: String) -> Result<(), LemmyError> {
// Setup the connections
let mut scheduler = Scheduler::new();
let mut conn = PgConnection::establish(&db_url).expect("could not establish connection");
let mut conn_2 = PgConnection::establish(&db_url).expect("could not establish connection");
active_counts(&mut conn);
update_banned_when_expired(&mut conn);
@ -33,6 +41,11 @@ pub fn setup(db_url: String) -> Result<(), LemmyError> {
clear_old_activities(&mut conn);
});
update_instance_software(&mut conn_2, &user_agent);
scheduler.every(1.days()).run(move || {
update_instance_software(&mut conn_2, &user_agent);
});
// Manually run the scheduler in an event loop
loop {
scheduler.run_pending();
@ -120,3 +133,67 @@ fn drop_ccnew_indexes(conn: &mut PgConnection) {
.execute(conn)
.expect("drop ccnew indexes");
}
/// Updates the instance software and version
fn update_instance_software(conn: &mut PgConnection, user_agent: &str) {
use lemmy_db_schema::schema::instance;
info!("Updating instances software and versions...");
let client = Client::builder()
.user_agent(user_agent)
.timeout(REQWEST_TIMEOUT)
.build()
.expect("couldnt build reqwest client");
let instances = instance::table
.get_results::<Instance>(conn)
.expect("no instances found");
for instance in instances {
let node_info_url = format!("https://{}/nodeinfo/2.0.json", instance.domain);
// Skip it if it can't connect
let res = client
.get(&node_info_url)
.send()
.ok()
.and_then(|t| t.json::<NodeInfo>().ok());
if let Some(node_info) = res {
let software = node_info.software.as_ref();
let form = InstanceForm::builder()
.domain(instance.domain)
.software(software.and_then(|s| s.name.clone()))
.version(software.and_then(|s| s.version.clone()))
.updated(Some(naive_now()))
.build();
diesel::update(instance::table.find(instance.id))
.set(form)
.execute(conn)
.expect("update site instance software");
}
}
info!("Done.");
}
#[cfg(test)]
mod tests {
use lemmy_routes::nodeinfo::NodeInfo;
use reqwest::Client;
#[tokio::test]
async fn test_nodeinfo() {
let client = Client::builder().build().unwrap();
let lemmy_ml_nodeinfo = client
.get("https://lemmy.ml/nodeinfo/2.0.json")
.send()
.await
.unwrap()
.json::<NodeInfo>()
.await
.unwrap();
assert_eq!(lemmy_ml_nodeinfo.software.unwrap().name.unwrap(), "lemmy");
}
}