Cache result of LocalSite::read to avoid unnecessary db calls (#4585)

* Cache result of LocalSite::read to avoid unnecessary db calls

* single const for cache duration

* clippy

* revert apub send changes

* clippy

* fmt
This commit is contained in:
Nutomic 2024-04-03 23:38:31 +02:00 committed by GitHub
parent aaaa362b98
commit 087684658a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 44 additions and 28 deletions

1
Cargo.lock generated
View file

@ -2772,6 +2772,7 @@ dependencies = [
"futures-util",
"i-love-jesus",
"lemmy_utils",
"moka",
"once_cell",
"pretty_assertions",
"regex",

View file

@ -42,21 +42,18 @@ use lemmy_utils::{
markdown::{markdown_check_for_blocked_urls, markdown_rewrite_image_links},
slurs::{build_slur_regex, remove_slurs},
},
CACHE_DURATION_SHORT,
};
use moka::future::Cache;
use once_cell::sync::Lazy;
use regex::{escape, Regex, RegexSet};
use rosetta_i18n::{Language, LanguageId};
use std::{collections::HashSet, time::Duration};
use std::collections::HashSet;
use tracing::warn;
use url::{ParseError, Url};
use urlencoding::encode;
pub static AUTH_COOKIE_NAME: &str = "jwt";
#[cfg(debug_assertions)]
static URL_BLOCKLIST_RECHECK_DELAY: Duration = Duration::from_millis(500);
#[cfg(not(debug_assertions))]
static URL_BLOCKLIST_RECHECK_DELAY: Duration = Duration::from_secs(60);
#[tracing::instrument(skip_all)]
pub async fn is_mod_or_admin(
@ -527,7 +524,7 @@ pub async fn get_url_blocklist(context: &LemmyContext) -> LemmyResult<RegexSet>
static URL_BLOCKLIST: Lazy<Cache<(), RegexSet>> = Lazy::new(|| {
Cache::builder()
.max_capacity(1)
.time_to_live(URL_BLOCKLIST_RECHECK_DELAY)
.time_to_live(CACHE_DURATION_SHORT)
.build()
});

View file

@ -20,11 +20,11 @@ use lemmy_db_views_actor::structs::{
};
use lemmy_utils::{
error::{LemmyError, LemmyErrorExt, LemmyErrorType},
CACHE_DURATION_SHORT,
VERSION,
};
use moka::future::Cache;
use once_cell::sync::Lazy;
use std::time::Duration;
#[tracing::instrument(skip(context))]
pub async fn get_site(
@ -34,7 +34,7 @@ pub async fn get_site(
static CACHE: Lazy<Cache<(), GetSiteResponse>> = Lazy::new(|| {
Cache::builder()
.max_capacity(1)
.time_to_live(Duration::from_secs(1))
.time_to_live(CACHE_DURATION_SHORT)
.build()
});

View file

@ -9,11 +9,14 @@ use lemmy_db_schema::{
source::{activity::ReceivedActivity, instance::Instance, local_site::LocalSite},
utils::{ActualDbPool, DbPool},
};
use lemmy_utils::error::{LemmyError, LemmyErrorType, LemmyResult};
use lemmy_utils::{
error::{LemmyError, LemmyErrorType, LemmyResult},
CACHE_DURATION_SHORT,
};
use moka::future::Cache;
use once_cell::sync::Lazy;
use serde_json::Value;
use std::{sync::Arc, time::Duration};
use std::sync::Arc;
use url::Url;
pub mod activities;
@ -27,11 +30,6 @@ pub mod objects;
pub mod protocol;
pub const FEDERATION_HTTP_FETCH_LIMIT: u32 = 50;
/// All incoming and outgoing federation actions read the blocklist/allowlist and slur filters
/// multiple times. This causes a huge number of database reads if we hit the db directly. So we
/// cache these values for a short time, which will already make a huge difference and ensures that
/// changes take effect quickly.
const BLOCKLIST_CACHE_DURATION: Duration = Duration::from_secs(60);
/// Only include a basic context to save space and bandwidth. The main context is hosted statically
/// on join-lemmy.org. Include activitystreams explicitly for better compat, but this could
@ -122,10 +120,14 @@ pub(crate) struct LocalSiteData {
pub(crate) async fn local_site_data_cached(
pool: &mut DbPool<'_>,
) -> LemmyResult<Arc<LocalSiteData>> {
// All incoming and outgoing federation actions read the blocklist/allowlist and slur filters
// multiple times. This causes a huge number of database reads if we hit the db directly. So we
// cache these values for a short time, which will already make a huge difference and ensures that
// changes take effect quickly.
static CACHE: Lazy<Cache<(), Arc<LocalSiteData>>> = Lazy::new(|| {
Cache::builder()
.max_capacity(1)
.time_to_live(BLOCKLIST_CACHE_DURATION)
.time_to_live(CACHE_DURATION_SHORT)
.build()
});
Ok(

View file

@ -80,6 +80,7 @@ rustls = { workspace = true, optional = true }
uuid = { workspace = true, features = ["v4"] }
i-love-jesus = { workspace = true, optional = true }
anyhow = { workspace = true }
moka.workspace = true
[dev-dependencies]
serial_test = { workspace = true }

View file

@ -5,6 +5,9 @@ use crate::{
};
use diesel::{dsl::insert_into, result::Error};
use diesel_async::RunQueryDsl;
use lemmy_utils::{error::LemmyError, CACHE_DURATION_SHORT};
use moka::future::Cache;
use once_cell::sync::Lazy;
impl LocalSite {
pub async fn create(pool: &mut DbPool<'_>, form: &LocalSiteInsertForm) -> Result<Self, Error> {
@ -14,9 +17,21 @@ impl LocalSite {
.get_result::<Self>(conn)
.await
}
pub async fn read(pool: &mut DbPool<'_>) -> Result<Self, Error> {
pub async fn read(pool: &mut DbPool<'_>) -> Result<Self, LemmyError> {
static CACHE: Lazy<Cache<(), LocalSite>> = Lazy::new(|| {
Cache::builder()
.max_capacity(1)
.time_to_live(CACHE_DURATION_SHORT)
.build()
});
Ok(
CACHE
.try_get_with((), async {
let conn = &mut get_conn(pool).await?;
local_site.first::<Self>(conn).await
})
.await?,
)
}
pub async fn update(pool: &mut DbPool<'_>, form: &LocalSiteUpdateForm) -> Result<Self, Error> {
let conn = &mut get_conn(pool).await?;

View file

@ -1,6 +1,7 @@
use anyhow::{anyhow, Context, Result};
use diesel::prelude::*;
use diesel_async::RunQueryDsl;
use lemmy_api_common::lemmy_utils::CACHE_DURATION_SHORT;
use lemmy_apub::{
activity_lists::SharedInboxActivities,
fetcher::{site_or_community_or_user::SiteOrCommunityOrUser, user_or_community::UserOrCommunity},
@ -31,6 +32,7 @@ pub(crate) static LEMMY_TEST_FAST_FEDERATION: Lazy<bool> = Lazy::new(|| {
.map(|s| !s.is_empty())
.unwrap_or(false)
});
/// Recheck for new federation work every n seconds.
///
/// When the queue is processed faster than new activities are added and it reaches the current time with an empty batch,
@ -167,15 +169,8 @@ pub(crate) async fn get_activity_cached(
/// return the most current activity id (with 1 second cache)
pub(crate) async fn get_latest_activity_id(pool: &mut DbPool<'_>) -> Result<ActivityId> {
static CACHE: Lazy<Cache<(), ActivityId>> = Lazy::new(|| {
Cache::builder()
.time_to_live(if *LEMMY_TEST_FAST_FEDERATION {
*WORK_FINISHED_RECHECK_DELAY
} else {
Duration::from_secs(1)
})
.build()
});
static CACHE: Lazy<Cache<(), ActivityId>> =
Lazy::new(|| Cache::builder().time_to_live(CACHE_DURATION_SHORT).build());
CACHE
.try_get_with((), async {
use diesel::dsl::max;

View file

@ -23,6 +23,11 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION");
pub const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
#[cfg(debug_assertions)]
pub const CACHE_DURATION_SHORT: Duration = Duration::from_millis(500);
#[cfg(not(debug_assertions))]
pub const CACHE_DURATION_SHORT: Duration = Duration::from_secs(60);
#[macro_export]
macro_rules! location_info {
() => {