diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
index af858fc46..3b266e59b 100644
--- a/lib/pleroma/search.ex
+++ b/lib/pleroma/search.ex
@@ -1,12 +1,12 @@
 defmodule Pleroma.Search do
   alias Pleroma.Workers.SearchIndexingWorker
 
-  def add_to_index(activity) do
-    SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id})
+  def add_to_index(%Pleroma.Activity{id: activity_id}) do
+    SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id})
   end
 
-  def remove_from_index(object) do
-    SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id})
+  def remove_from_index(%Pleroma.Object{id: object_id}) do
+    SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id})
   end
 
   def search(query, options) do
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 4762b5ac6..644e62630 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -330,7 +330,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     if result == :ok do
       Notification.create_notifications(object)
 
-      Pleroma.Search.remove_from_index(deleted_object)
+      # Only remove from index when deleting actual objects, not users or anything else
+      with %Pleroma.Object{} <- deleted_object do
+        Pleroma.Search.remove_from_index(deleted_object)
+      end
 
       {:ok, object, meta}
     else