Merge 723045a32a into 7c146272c3

.gitignore

@@ -34,3 +34,6 @@ dev_pgdata/
 
 # database dumps
 *.sqldump
+
+# compiled example plugin
+plugins/plugin.wasm

.woodpecker.yml

@@ -229,7 +229,7 @@ steps:
       DO_WRITE_HOSTS_FILE: "1"
     commands:
       - *install_pnpm
-      - apt update && apt install -y bash curl postgresql-client
+      - apt update && apt install -y bash curl postgresql-client golang
       - bash api_tests/prepare-drone-federation-test.sh
       - cd api_tests/
       - pnpm i

Cargo.toml

@@ -204,7 +204,9 @@ chrono = { workspace = true }
 prometheus = { version = "0.13.3", features = ["process"] }
 serial_test = { workspace = true }
 clap = { workspace = true }
+serde.workspace = true
 actix-web-prom = "0.7.0"
+actix-http = "3.6.0"
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }

api_tests/prepare-drone-federation-test.sh

@@ -49,8 +49,6 @@ else
   done
 fi
 
-echo "$PWD"
-
 LOG_DIR=target/log
 mkdir -p $LOG_DIR
 
@@ -75,6 +73,12 @@ LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_delta.hjson \
   LEMMY_DATABASE_URL="${LEMMY_DATABASE_URL}/lemmy_delta" \
   target/lemmy_server >$LOG_DIR/lemmy_delta.out 2>&1 &
 
+# plugin setup
+pushd plugins
+# need to use tinygo because apparently go has only experimental support for wasm target
+GOPATH=$HOME/.local/share/go tinygo build -o plugin.wasm -target wasi main.go
+popd
+
 echo "start epsilon"
 # An instance who has a blocklist, with lemmy-alpha blocked
 LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_epsilon.hjson \

api_tests/run-federation-test.sh

@@ -11,7 +11,7 @@ killall -s1 lemmy_server || true
 popd
 
 pnpm i
-pnpm api-test || true
+pnpm api-test-post || true
 
 killall -s1 lemmy_server || true
 killall -s1 pict-rs || true

api_tests/src/post.spec.ts

@@ -763,3 +763,15 @@ test("Fetch post with redirect", async () => {
   let gammaPost2 = await gamma.resolveObject(form);
   expect(gammaPost2.post).toBeDefined();
 });
+
+test.only("Plugin test", async () => {
+  let community = await createCommunity(epsilon);
+  let postRes = createPost(
+    epsilon,
+    community.community_view.community.id,
+    "https://example.com/",
+    "body",
+    "foobar",
+  );
+  expect((await postRes).post_view.post.name).toBe("Hello plugin!");
+});

crates/apub/Cargo.toml

@@ -47,6 +47,10 @@ html2md = "0.2.14"
 html2text = "0.6.0"
 stringreader = "0.1.1"
 enum_delegate = "0.2.0"
+extism = { version = "1.2.0", features = [
+  "register-filesystem",
+], default-features = false }
+extism-convert = { version = "1.2.0", default-features = false }
 
 [dev-dependencies]
 serial_test = { workspace = true }

crates/apub/src/lib.rs

@@ -27,6 +27,7 @@ pub mod fetcher;
 pub mod http;
 pub(crate) mod mentions;
 pub mod objects;
+pub mod plugins;
 pub mod protocol;
 
 pub const FEDERATION_HTTP_FETCH_LIMIT: u32 = 50;

crates/apub/src/objects/post.rs

@@ -3,6 +3,7 @@ use crate::{
   check_apub_id_valid_with_strictness,
   local_site_data_cached,
   objects::{read_from_string_or_source_opt, verify_is_remote_object},
+  plugins::{call_plugin, load_plugins},
   protocol::{
     objects::{
       page::{Attachment, AttributedTo, Hashtag, HashtagType, Page, PageType},
@@ -50,6 +51,7 @@ use lemmy_utils::{
 };
 use std::ops::Deref;
 use stringreader::StringReader;
+use tracing::info;
 use url::Url;
 
 const MAX_TITLE_LENGTH: usize = 200;
@@ -224,7 +226,7 @@ impl Object for ApubPost {
     let first_attachment = page.attachment.first();
     let local_site = LocalSite::read(&mut context.pool()).await.ok();
 
-    let form = if !page.is_mod_action(context).await? {
+    let mut form = if !page.is_mod_action(context).await? {
       let url = if let Some(attachment) = first_attachment.cloned() {
         Some(attachment.url())
       } else if page.kind == PageType::Video {
@@ -275,8 +277,25 @@ impl Object for ApubPost {
         .build()
     };
 
+    // TODO: move this all into helper function
+    let before_plugin_hook = "federation_before_receive_post";
+    info!("Calling plugin hook {}", &before_plugin_hook);
+    if let Some(mut plugins) = load_plugins()? {
+      if plugins.function_exists(&before_plugin_hook) {
+        call_plugin(plugins, &before_plugin_hook, &mut form)?;
+      }
+    }
+
     let timestamp = page.updated.or(page.published).unwrap_or_else(naive_now);
-    let post = Post::insert_apub(&mut context.pool(), timestamp, &form).await?;
+    let mut post = Post::insert_apub(&mut context.pool(), timestamp, &form).await?;
+
+    let after_plugin_hook = "federation_after_receive_post";
+    info!("Calling plugin hook {}", &after_plugin_hook);
+    if let Some(mut plugins) = load_plugins()? {
+      if plugins.function_exists(&after_plugin_hook) {
+        call_plugin(plugins, &after_plugin_hook, &mut post)?;
+      }
+    }
 
     generate_post_link_metadata(
       post.clone(),

crates/apub/src/plugins.rs

@@ -0,0 +1,39 @@
+use extism::{Manifest, Plugin};
+use lemmy_utils::{error::LemmyResult, LemmyErrorType};
+use serde::{Deserialize, Serialize};
+use std::{ffi::OsStr, fs::read_dir};
+
+pub fn load_plugins() -> LemmyResult<Option<Plugin>> {
+  // TODO: make dir configurable via env var
+  // TODO: should only read fs once at startup for performance
+  let plugin_paths = read_dir("plugins")?;
+
+  let mut wasm_files = vec![];
+  for path in plugin_paths {
+    let path = path?.path();
+    if path.extension() == Some(OsStr::new("wasm")) {
+      wasm_files.push(path);
+    }
+  }
+  if !wasm_files.is_empty() {
+    // TODO: what if theres more than one plugin for the same hook?
+    let manifest = Manifest::new(wasm_files);
+    let plugin = Plugin::new(manifest, [], true)?;
+    Ok(Some(plugin))
+  } else {
+    Ok(None)
+  }
+}
+
+pub fn call_plugin<T: Serialize + for<'de> Deserialize<'de> + Clone>(
+  mut plugins: Plugin,
+  name: &str,
+  data: &mut T,
+) -> LemmyResult<()> {
+  *data = plugins
+    .call::<extism_convert::Json<T>, extism_convert::Json<T>>(name, (*data).clone().into())
+    .map_err(|e| LemmyErrorType::PluginError(e.to_string()))?
+    .0
+    .into();
+  Ok(())
+}

crates/db_schema/src/source/post.rs

@@ -60,7 +60,7 @@ pub struct Post {
   pub alt_text: Option<String>,
 }
 
-#[derive(Debug, Clone, TypedBuilder)]
+#[derive(Debug, Clone, TypedBuilder, Serialize, Deserialize)]
 #[builder(field_defaults(default))]
 #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
 #[cfg_attr(feature = "full", diesel(table_name = post))]

crates/utils/src/error.rs

@@ -175,6 +175,7 @@ pub enum LemmyErrorType {
   InvalidBotAction,
   CantBlockLocalInstance,
   UrlWithoutDomain,
+  PluginError(String),
   Unknown(String),
 }
 

plugins/go.mod

@@ -0,0 +1,5 @@
+module example_plugin
+
+go 1.22.0
+
+require github.com/extism/go-pdk v1.0.2 // indirect

plugins/go.sum

@@ -0,0 +1,2 @@
+github.com/extism/go-pdk v1.0.2 h1:UB7oTW3tw2zoMlsUdBEDAAbhQg9OudzgNeyCwQYZ730=
+github.com/extism/go-pdk v1.0.2/go.mod h1:Gz+LIU/YCKnKXhgge8yo5Yu1F/lbv7KtKFkiCSzW/P4=

plugins/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/extism/go-pdk"
+)
+
+type CreatePost struct {
+	Name string `json:"name"`
+	Body *string `json:"body,omitempty"`
+	Community_id int32 `json:"community_id"`
+	Url *string `json:"url,omitempty"`
+	Alt_text *string `json:"alt_text,omitempty"`
+	Honeypot *string `json:"honeypot,omitempty"`
+	Nsfw *bool `json:"nsfw,omitempty"`
+	Language_id *int32 `json:"language_id,omitempty"`
+	Custom_thumbnail *string `json:"custom_thumbnail,omitempty"`
+  }
+
+//export api_before_post_post
+func api_before_post_post() int32 {
+	params := CreatePost{}
+	// use json input helper, which automatically unmarshals the plugin input into your struct
+	err := pdk.InputJSON(&params)
+	if err != nil {
+		pdk.SetError(err)
+		return 1
+	}
+	if params.Name == "foobar" {
+		params.Name = "Hello plugin!"
+	}
+	// use json output helper, which automatically marshals your struct to the plugin output
+	err = pdk.OutputJSON(params)
+	if err != nil {
+		pdk.SetError(err)
+		return 1
+	}
+	return 0
+}
+
+func main() {}

src/api_routes_http.rs

@@ -1,3 +1,4 @@
+use crate::plugin_middleware::PluginMiddleware;
 use actix_web::{guard, web};
 use lemmy_api::{
   comment::{
@@ -139,6 +140,7 @@ use lemmy_utils::rate_limit::RateLimitCell;
 pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) {
   cfg.service(
     web::scope("/api/v3")
+      .wrap(PluginMiddleware::new())
       .route("/image_proxy", web::get().to(image_proxy))
       // Site
       .service(

src/lib.rs

@@ -1,5 +1,6 @@
 pub mod api_routes_http;
 pub mod code_migrations;
+pub mod plugin_middleware;
 pub mod prometheus_metrics;
 pub mod root_span_builder;
 pub mod scheduled_tasks;

src/plugin_middleware.rs

@@ -0,0 +1,97 @@
+use actix_http::{body::BoxBody, h1::Payload};
+use actix_web::{
+  body::MessageBody,
+  dev::{forward_ready, Service, ServiceRequest, ServiceResponse, Transform},
+  web::Bytes,
+  Error,
+};
+use core::future::Ready;
+use futures_util::future::LocalBoxFuture;
+use lemmy_apub::plugins::{call_plugin, load_plugins};
+use serde_json::Value;
+use std::{future::ready, rc::Rc};
+use tracing::info;
+
+#[derive(Clone)]
+pub struct PluginMiddleware {}
+
+impl PluginMiddleware {
+  pub fn new() -> Self {
+    PluginMiddleware {}
+  }
+}
+impl<S> Transform<S, ServiceRequest> for PluginMiddleware
+where
+  S: Service<ServiceRequest, Response = ServiceResponse<BoxBody>, Error = Error> + 'static,
+  S::Future: 'static,
+{
+  type Response = ServiceResponse<BoxBody>;
+  type Error = Error;
+  type Transform = SessionService<S>;
+  type InitError = ();
+  type Future = Ready<Result<Self::Transform, Self::InitError>>;
+
+  fn new_transform(&self, service: S) -> Self::Future {
+    ready(Ok(SessionService {
+      service: Rc::new(service),
+    }))
+  }
+}
+
+pub struct SessionService<S> {
+  service: Rc<S>,
+}
+
+impl<S> Service<ServiceRequest> for SessionService<S>
+where
+  S: Service<ServiceRequest, Response = ServiceResponse<BoxBody>, Error = Error> + 'static,
+  S::Future: 'static,
+{
+  type Response = ServiceResponse<BoxBody>;
+  type Error = Error;
+  type Future = LocalBoxFuture<'static, Result<Self::Response, Self::Error>>;
+
+  forward_ready!(service);
+
+  fn call(&self, mut service_req: ServiceRequest) -> Self::Future {
+    let svc = self.service.clone();
+
+    Box::pin(async move {
+      let method = service_req.method().clone();
+      let path = service_req.path().replace("/api/v3/", "").replace("/", "_");
+      // TODO: naming can be a bit silly, `POST /api/v3/post` becomes `api_before_post_post`
+      let before_plugin_hook = format!("api_before_{method}_{path}").to_lowercase();
+
+      info!("Calling plugin hook {}", &before_plugin_hook);
+      if let Some(mut plugins) = load_plugins()? {
+        if plugins.function_exists(&before_plugin_hook) {
+          let payload = service_req.extract::<Bytes>().await?;
+
+          let mut json: Value = serde_json::from_slice(&payload.to_vec())?;
+          call_plugin(plugins, &before_plugin_hook, &mut json)?;
+
+          let (_, mut new_payload) = Payload::create(true);
+          new_payload.unread_data(Bytes::from(serde_json::to_vec(&json)?));
+          service_req.set_payload(new_payload.into());
+        }
+      }
+      let mut res = svc.call(service_req).await?;
+
+      // TODO: add after hook
+      let after_plugin_hook = format!("api_after_{method}_{path}").to_lowercase();
+      info!("Calling plugin hook {}", &after_plugin_hook);
+      if let Some(mut plugins) = load_plugins()? {
+        if plugins.function_exists(&before_plugin_hook) {
+          res = res.map_body(|_, body| {
+            let mut json: Value =
+              serde_json::from_slice(&body.try_into_bytes().unwrap().to_vec()).unwrap();
+            call_plugin(plugins, &after_plugin_hook, &mut json).unwrap();
+            BoxBody::new(Bytes::from(serde_json::to_vec(&json).unwrap()))
+          });
+        }
+      }
+
+      Ok(res)
+    })
+  }
+}