From b167364c5c936cd98f65c165a876057278515b8a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 20 Feb 2023 12:58:41 -0800 Subject: [PATCH] Use a separate queue for broadcasts I think this will go a long way to solve the federation delay problems we're seeing on b.s. I'm not sure at what point adding more queues will create more problems than it solves, but I do think in this case the queues are out of balance and moving broadcasts (which are the most common type of `medium_priority` task at the moment) to their own queue will be an improvement. --- bookwyrm/models/activitypub_mixin.py | 8 ++++---- bookwyrm/tasks.py | 2 ++ bookwyrm/templates/settings/celery.html | 16 +++++++++++----- bookwyrm/views/admin/celery_status.py | 11 ++++++----- contrib/systemd/bookwyrm-worker.service | 2 +- docker-compose.yml | 2 +- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index 9361854ba..ec1c34a40 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -21,7 +21,7 @@ from django.utils.http import http_date from bookwyrm import activitypub from bookwyrm.settings import USER_AGENT, PAGE_LENGTH from bookwyrm.signatures import make_signature, make_digest -from bookwyrm.tasks import app, MEDIUM +from bookwyrm.tasks import app, MEDIUM, BROADCAST from bookwyrm.models.fields import ImageField, ManyToManyField logger = logging.getLogger(__name__) @@ -126,7 +126,7 @@ class ActivitypubMixin: # there OUGHT to be only one match return match.first() - def broadcast(self, activity, sender, software=None, queue=MEDIUM): + def broadcast(self, activity, sender, software=None, queue=BROADCAST): """send out an activity""" broadcast_task.apply_async( args=( @@ -198,7 +198,7 @@ class ActivitypubMixin: class ObjectMixin(ActivitypubMixin): """add this mixin for object models that are AP serializable""" - def save(self, *args, created=None, software=None, priority=MEDIUM, **kwargs): + def save(self, *args, created=None, software=None, priority=BROADCAST, **kwargs): """broadcast created/updated/deleted objects as appropriate""" broadcast = kwargs.get("broadcast", True) # this bonus kwarg would cause an error in the base save method @@ -506,7 +506,7 @@ def unfurl_related_field(related_field, sort_field=None): return related_field.remote_id -@app.task(queue=MEDIUM) +@app.task(queue=BROADCAST) def broadcast_task(sender_id: int, activity: str, recipients: List[str]): """the celery task for broadcast""" user_model = apps.get_model("bookwyrm.User", require_ready=True) diff --git a/bookwyrm/tasks.py b/bookwyrm/tasks.py index ec018e179..91977afda 100644 --- a/bookwyrm/tasks.py +++ b/bookwyrm/tasks.py @@ -16,3 +16,5 @@ MEDIUM = "medium_priority" HIGH = "high_priority" # import items get their own queue because they're such a pain in the ass IMPORTS = "imports" +# I keep making more queues?? this one broadcasting out +BROADCAST = "broadcast" diff --git a/bookwyrm/templates/settings/celery.html b/bookwyrm/templates/settings/celery.html index 6a3ee6574..65315da01 100644 --- a/bookwyrm/templates/settings/celery.html +++ b/bookwyrm/templates/settings/celery.html @@ -20,31 +20,37 @@ {% if queues %}

{% trans "Queues" %}

-
-
+
+

{% trans "Low priority" %}

{{ queues.low_priority|intcomma }}

-
+

{% trans "Medium priority" %}

{{ queues.medium_priority|intcomma }}

-
+

{% trans "High priority" %}

{{ queues.high_priority|intcomma }}

-
+

{% trans "Imports" %}

{{ queues.imports|intcomma }}

+
+
+

{% trans "Broadcasts" %}

+

{{ queues.broadcast|intcomma }}

+
+
{% else %} diff --git a/bookwyrm/views/admin/celery_status.py b/bookwyrm/views/admin/celery_status.py index 8c6b9f5a3..e0b1fe18c 100644 --- a/bookwyrm/views/admin/celery_status.py +++ b/bookwyrm/views/admin/celery_status.py @@ -8,7 +8,7 @@ from django.views.decorators.http import require_GET import redis from celerywyrm import settings -from bookwyrm.tasks import app as celery +from bookwyrm.tasks import app as celery, LOW, MEDIUM, HIGH, IMPORTS, BROADCAST r = redis.from_url(settings.REDIS_BROKER_URL) @@ -35,10 +35,11 @@ class CeleryStatus(View): try: queues = { - "low_priority": r.llen("low_priority"), - "medium_priority": r.llen("medium_priority"), - "high_priority": r.llen("high_priority"), - "imports": r.llen("imports"), + LOW: r.llen(LOW), + MEDIUM: r.llen(MEDIUM), + HIGH: r.llen(HIGH), + IMPORTS: r.llen(IMPORTS), + BROADCAST: r.llen(BROADCAST), } # pylint: disable=broad-except except Exception as err: diff --git a/contrib/systemd/bookwyrm-worker.service b/contrib/systemd/bookwyrm-worker.service index d79fdabf6..e4f1e98b5 100644 --- a/contrib/systemd/bookwyrm-worker.service +++ b/contrib/systemd/bookwyrm-worker.service @@ -6,7 +6,7 @@ After=network.target postgresql.service redis.service User=bookwyrm Group=bookwyrm WorkingDirectory=/opt/bookwyrm/ -ExecStart=/opt/bookwyrm/venv/bin/celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority,import +ExecStart=/opt/bookwyrm/venv/bin/celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority,import,broadcast StandardOutput=journal StandardError=inherit diff --git a/docker-compose.yml b/docker-compose.yml index 4ab7a1c34..4389c08aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,7 +62,7 @@ services: build: . networks: - main - command: celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority,imports + command: celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority,imports,broadcast volumes: - .:/app - static_volume:/app/static