From 0bb4b0d71dfe14c60f9be38fe574b5f181a32995 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 8 Nov 2023 15:24:47 -0800 Subject: [PATCH 001/123] Changes to how images are serialized I'm just going to see if any tests fail? --- bookwyrm/activitypub/person.py | 2 +- bookwyrm/models/fields.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py index 85cf44409..139d6061d 100644 --- a/bookwyrm/activitypub/person.py +++ b/bookwyrm/activitypub/person.py @@ -35,7 +35,7 @@ class Person(ActivityObject): endpoints: Dict = None name: str = None summary: str = None - icon: Image = field(default_factory=lambda: {}) + icon: Image = None bookwyrmUser: bool = False manuallyApprovesFollowers: str = False discoverable: str = False diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 1e458c815..6c8a130f8 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -477,7 +477,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): if not url: return None - return activitypub.Document(url=url, name=alt) + return activitypub.Image(url=url, name=alt) def field_from_activity(self, value, allow_external_connections=True): image_slug = value From 44b14f4933e80222a1b75dcdaf8ad236810280f8 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 8 Nov 2023 16:00:10 -0800 Subject: [PATCH 002/123] Fixes workflow errors --- bookwyrm/activitypub/person.py | 2 +- bookwyrm/tests/models/test_fields.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py index 139d6061d..dfec92e4c 100644 --- a/bookwyrm/activitypub/person.py +++ b/bookwyrm/activitypub/person.py @@ -1,5 +1,5 @@ """ actor serializer """ -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Dict from .base_activity import ActivityObject diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index 553a533d5..c27d144d7 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -436,7 +436,7 @@ class ModelFields(TestCase): ) ) self.assertEqual(output.name, "") - self.assertEqual(output.type, "Document") + self.assertEqual(output.type, "Image") @responses.activate def test_image_field_from_activity(self, *_): From 7684101f1591292c15be7d80b87d51937e21ff43 Mon Sep 17 00:00:00 2001 From: Matt Lehrer Date: Thu, 16 Nov 2023 10:38:41 +0100 Subject: [PATCH 003/123] move ratings to new edition --- bookwyrm/tests/views/books/test_editions.py | 28 +++++++++++++++++++++ bookwyrm/views/books/editions.py | 7 ++++++ 2 files changed, 35 insertions(+) diff --git a/bookwyrm/tests/views/books/test_editions.py b/bookwyrm/tests/views/books/test_editions.py index 70a95051a..a40d97b57 100644 --- a/bookwyrm/tests/views/books/test_editions.py +++ b/bookwyrm/tests/views/books/test_editions.py @@ -129,3 +129,31 @@ class BookViews(TestCase): self.assertEqual(models.ShelfBook.objects.get().book, edition2) self.assertEqual(models.ReadThrough.objects.get().book, edition2) + + @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") + @patch("bookwyrm.activitystreams.populate_stream_task.delay") + @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") + def test_move_ratings_on_switch_edition(self, *_): + """updates user's rating on a book to new edition""" + work = models.Work.objects.create(title="test work") + edition1 = models.Edition.objects.create(title="first ed", parent_work=work) + edition2 = models.Edition.objects.create(title="second ed", parent_work=work) + + models.ReviewRating.objects.create( + book=edition1, + user=self.local_user, + rating=3, + ) + + self.assertIsInstance(models.ReviewRating.objects.get(user=self.local_user, book=edition1), models.ReviewRating) + with self.assertRaises(models.ReviewRating.DoesNotExist): + models.ReviewRating.objects.get(user=self.local_user, book=edition2) + + request = self.factory.post("", {"edition": edition2.id}) + request.user = self.local_user + views.switch_edition(request) + + self.assertIsInstance(models.ReviewRating.objects.get(user=self.local_user, book=edition2), models.ReviewRating) + with self.assertRaises(models.ReviewRating.DoesNotExist): + models.ReviewRating.objects.get(user=self.local_user, book=edition1) diff --git a/bookwyrm/views/books/editions.py b/bookwyrm/views/books/editions.py index 54d1bd84c..e34b045bb 100644 --- a/bookwyrm/views/books/editions.py +++ b/bookwyrm/views/books/editions.py @@ -103,4 +103,11 @@ def switch_edition(request): readthrough.book = new_edition readthrough.save() + ratings = models.ReviewRating.objects.filter( + book__parent_work=new_edition.parent_work, user=request.user + ) + for rating in ratings.all(): + rating.book = new_edition + rating.save() + return redirect(f"/book/{new_edition.id}") From bd920a4630975f3e5adac6c078dc1a5eaf9a48f6 Mon Sep 17 00:00:00 2001 From: Matt Lehrer Date: Thu, 16 Nov 2023 10:38:45 +0100 Subject: [PATCH 004/123] move reviews to new edition --- bookwyrm/tests/views/books/test_editions.py | 30 +++++++++++++++++++++ bookwyrm/views/books/editions.py | 7 +++++ 2 files changed, 37 insertions(+) diff --git a/bookwyrm/tests/views/books/test_editions.py b/bookwyrm/tests/views/books/test_editions.py index a40d97b57..bbae517ac 100644 --- a/bookwyrm/tests/views/books/test_editions.py +++ b/bookwyrm/tests/views/books/test_editions.py @@ -157,3 +157,33 @@ class BookViews(TestCase): self.assertIsInstance(models.ReviewRating.objects.get(user=self.local_user, book=edition2), models.ReviewRating) with self.assertRaises(models.ReviewRating.DoesNotExist): models.ReviewRating.objects.get(user=self.local_user, book=edition1) + + @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") + @patch("bookwyrm.activitystreams.populate_stream_task.delay") + @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") + def test_move_reviews_on_switch_edition(self, *_): + """updates user's review on a book to new edition""" + work = models.Work.objects.create(title="test work") + edition1 = models.Edition.objects.create(title="first ed", parent_work=work) + edition2 = models.Edition.objects.create(title="second ed", parent_work=work) + + models.Review.objects.create( + book=edition1, + user=self.local_user, + name="blah", + rating=3, + content="not bad" + ) + + self.assertIsInstance(models.Review.objects.get(user=self.local_user, book=edition1), models.Review) + with self.assertRaises(models.Review.DoesNotExist): + models.Review.objects.get(user=self.local_user, book=edition2) + + request = self.factory.post("", {"edition": edition2.id}) + request.user = self.local_user + views.switch_edition(request) + + self.assertIsInstance(models.Review.objects.get(user=self.local_user, book=edition2), models.Review) + with self.assertRaises(models.Review.DoesNotExist): + models.Review.objects.get(user=self.local_user, book=edition1) diff --git a/bookwyrm/views/books/editions.py b/bookwyrm/views/books/editions.py index e34b045bb..f061caab8 100644 --- a/bookwyrm/views/books/editions.py +++ b/bookwyrm/views/books/editions.py @@ -110,4 +110,11 @@ def switch_edition(request): rating.book = new_edition rating.save() + reviews = models.Review.objects.filter( + book__parent_work=new_edition.parent_work, user=request.user + ) + for review in reviews.all(): + review.book = new_edition + review.save() + return redirect(f"/book/{new_edition.id}") From f4da9fbf34fdf65d94315847022a00d08ef01a16 Mon Sep 17 00:00:00 2001 From: Matt Lehrer Date: Thu, 16 Nov 2023 20:37:46 +0100 Subject: [PATCH 005/123] remove unnecessary loop. ReviewRatings are a subclass and are included in the models.Review block --- bookwyrm/views/books/editions.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/bookwyrm/views/books/editions.py b/bookwyrm/views/books/editions.py index f061caab8..2bbb1c579 100644 --- a/bookwyrm/views/books/editions.py +++ b/bookwyrm/views/books/editions.py @@ -103,17 +103,12 @@ def switch_edition(request): readthrough.book = new_edition readthrough.save() - ratings = models.ReviewRating.objects.filter( - book__parent_work=new_edition.parent_work, user=request.user - ) - for rating in ratings.all(): - rating.book = new_edition - rating.save() - reviews = models.Review.objects.filter( book__parent_work=new_edition.parent_work, user=request.user ) for review in reviews.all(): + # because ratings are a subclass of reviews, + # this will pick up both ratings and reviews review.book = new_edition review.save() From 0299f2e2352773c53155ae815cd0a525d6408fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Fri, 24 Nov 2023 21:18:43 -0300 Subject: [PATCH 006/123] Add functional tests for search_vector triggers As metadata changes, search continues to work. --- bookwyrm/tests/test_book_search.py | 83 ++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index db6ba8353..2baff9bf1 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -134,3 +134,86 @@ class BookSearch(TestCase): # there's really not much to test here, it's just a dataclass self.assertEqual(result.confidence, 1) self.assertEqual(result.title, "Title") + + +class SearchVectorTriggers(TestCase): + """look for books as they change""" + + def setUp(self): + """we need basic test data and mocks""" + self.work = models.Work.objects.create(title="This Work") + self.author = models.Author.objects.create(name="Name") + self.edition = models.Edition.objects.create( + title="First Edition of Work", + subtitle="Some Extra Words Are Good", + series="A Fabulous Sequence of Items", + parent_work=self.work, + isbn_10="0000000000", + ) + self.edition.authors.add(self.author) + self.edition.save(broadcast=False) + + def test_search_after_changed_metadata(self): + """book found after updating metadata""" + self.assertEqual(self.edition, self._search_first("First")) # title + self.assertEqual(self.edition, self._search_first("Good")) # subtitle + self.assertEqual(self.edition, self._search_first("Sequence")) # series + + self.edition.title = "Second Title of Work" + self.edition.subtitle = "Fewer Words Is Better" + self.edition.series = "A Wondrous Bunch" + self.edition.save(broadcast=False) + + self.assertEqual(self.edition, self._search_first("Second")) # title new + self.assertEqual(self.edition, self._search_first("Fewer")) # subtitle new + self.assertEqual(self.edition, self._search_first("Wondrous")) # series new + + self.assertFalse(self._search_first("First")) # title old + self.assertFalse(self._search_first("Good")) # subtitle old + self.assertFalse(self._search_first("Sequence")) # series old + + def test_search_after_author_remove(self): + """book not found via removed author""" + self.assertEqual(self.edition, self._search_first("Name")) + + self.edition.authors.set([]) + self.edition.save(broadcast=False) + + self.assertFalse(self._search("Name")) + self.assertEqual(self.edition, self._search_first("Edition")) + + def test_search_after_author_add(self): + """book found by newly-added author""" + new_author = models.Author.objects.create(name="Mozilla") + + self.assertFalse(self._search("Mozilla")) + + self.edition.authors.add(new_author) + self.edition.save(broadcast=False) + + self.assertEqual(self.edition, self._search_first("Mozilla")) + self.assertEqual(self.edition, self._search_first("Name")) + + def test_search_after_updated_author_name(self): + """book found under new author name""" + self.assertEqual(self.edition, self._search_first("Name")) + self.assertFalse(self._search("Identifier")) + + self.author.name = "Identifier" + self.author.save(broadcast=False) + self.edition.refresh_from_db() + + self.assertFalse(self._search("Name")) + self.assertEqual(self.edition, self._search_first("Identifier")) + self.assertEqual(self.edition, self._search_first("Work")) + + def _search_first(self, query): + """wrapper around search_title_author""" + return self._search(query, return_first=True) + + # pylint: disable-next=no-self-use + def _search(self, query, *, return_first=False): + """wrapper around search_title_author""" + return book_search.search_title_author( + query, min_confidence=0, return_first=return_first + ) From e4d688665c7d54912b73752a270c4d4ead2a63b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Fri, 24 Nov 2023 14:15:00 -0300 Subject: [PATCH 007/123] Remove index for `author.search_vector`, which is never used --- bookwyrm/migrations/0190_book_search_updates.py | 16 ++++++++++++++++ bookwyrm/models/author.py | 6 ------ 2 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 bookwyrm/migrations/0190_book_search_updates.py diff --git a/bookwyrm/migrations/0190_book_search_updates.py b/bookwyrm/migrations/0190_book_search_updates.py new file mode 100644 index 000000000..52d80fcb9 --- /dev/null +++ b/bookwyrm/migrations/0190_book_search_updates.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.20 on 2023-11-24 17:11 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("bookwyrm", "0188_theme_loads"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="author", + name="bookwyrm_au_search__b050a8_gin", + ), + ] diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 981e3c0cc..b4488d46a 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -2,7 +2,6 @@ import re from typing import Tuple, Any -from django.contrib.postgres.indexes import GinIndex from django.db import models from bookwyrm import activitypub @@ -68,8 +67,3 @@ class Author(BookDataModel): return f"https://{DOMAIN}/author/{self.id}" activity_serializer = activitypub.Author - - class Meta: - """sets up postgres GIN index field""" - - indexes = (GinIndex(fields=["search_vector"]),) From 44ef928c3ceb8ce9b4426113551baa53563630f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 25 Nov 2023 16:08:36 -0300 Subject: [PATCH 008/123] Alter object row IDs to force test failure in original code --- bookwyrm/tests/test_book_search.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index 2baff9bf1..e66ea97be 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -1,5 +1,7 @@ """ test searching for books """ import datetime +from unittest import expectedFailure +from django.db import connection from django.test import TestCase from django.utils import timezone @@ -153,6 +155,17 @@ class SearchVectorTriggers(TestCase): self.edition.authors.add(self.author) self.edition.save(broadcast=False) + @classmethod + def setUpTestData(cls): + """create conditions that trigger known old bugs""" + with connection.cursor() as cursor: + cursor.execute( + """ + ALTER SEQUENCE bookwyrm_author_id_seq RESTART WITH 20; + ALTER SEQUENCE bookwyrm_book_authors_id_seq RESTART WITH 300; + """ + ) + def test_search_after_changed_metadata(self): """book found after updating metadata""" self.assertEqual(self.edition, self._search_first("First")) # title @@ -194,6 +207,7 @@ class SearchVectorTriggers(TestCase): self.assertEqual(self.edition, self._search_first("Mozilla")) self.assertEqual(self.edition, self._search_first("Name")) + @expectedFailure def test_search_after_updated_author_name(self): """book found under new author name""" self.assertEqual(self.edition, self._search_first("Name")) From 416a6caf2d4124b884fd1a04841908e572d92c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Fri, 24 Nov 2023 22:26:13 -0300 Subject: [PATCH 009/123] Define `author_search_vector_trigger` via Author.Meta.triggers Previously, triggers lived only in a particular migration file. With this change, code for the triggers resides in the model, and their lifecycle is managed through normal Django migrations. --- ...grate_search_vec_triggers_to_pgtriggers.py | 50 +++++++++++++++++++ bookwyrm/models/author.py | 29 +++++++++++ bookwyrm/settings.py | 1 + bookwyrm/utils/db.py | 22 ++++++++ requirements.txt | 1 + 5 files changed, 103 insertions(+) create mode 100644 bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py create mode 100644 bookwyrm/utils/db.py diff --git a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py new file mode 100644 index 000000000..10354fa67 --- /dev/null +++ b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py @@ -0,0 +1,50 @@ +# Generated by Django 3.2.20 on 2023-11-25 00:47 + +from importlib import import_module +import re + +from django.db import migrations +import pgtrigger.compiler +import pgtrigger.migrations + +trigger_migration = import_module("bookwyrm.migrations.0077_auto_20210623_2155") + +# it's _very_ convenient for development that this migration be reversible +search_vector_trigger = trigger_migration.Migration.operations[4] +author_search_vector_trigger = trigger_migration.Migration.operations[5] + + +assert re.search(r"\bCREATE TRIGGER search_vector_trigger\b", search_vector_trigger.sql) +assert re.search( + r"\bCREATE TRIGGER author_search_vector_trigger\b", + author_search_vector_trigger.sql, +) + + +class Migration(migrations.Migration): + dependencies = [ + ("bookwyrm", "0190_book_search_updates"), + ] + + operations = [ + pgtrigger.migrations.AddTrigger( + model_name="author", + trigger=pgtrigger.compiler.Trigger( + name="reset_search_vector_on_author_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="WITH book AS (SELECT bookwyrm_book.id AS row_id FROM bookwyrm_author LEFT OUTER JOIN bookwyrm_book_authors ON bookwyrm_book_authors.id = new.id LEFT OUTER JOIN bookwyrm_book ON bookwyrm_book.id = bookwyrm_book_authors.book_id) UPDATE bookwyrm_book SET search_vector = '' FROM book WHERE id = book.row_id;RETURN NEW;", + hash="9c0a472e2bf60e63d593cce49f47972c7b227a00", + operation='UPDATE OF "name"', + pgid="pgtrigger_reset_search_vector_on_author_edit_a447c", + table="bookwyrm_author", + when="AFTER", + ), + ), + ), + migrations.RunSQL( + sql="""DROP TRIGGER IF EXISTS author_search_vector_trigger ON bookwyrm_author; + DROP FUNCTION IF EXISTS author_trigger; + """, + reverse_sql=author_search_vector_trigger.sql, + ), + ] diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index b4488d46a..9b4f3c1bd 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -3,9 +3,11 @@ import re from typing import Tuple, Any from django.db import models +import pgtrigger from bookwyrm import activitypub from bookwyrm.settings import DOMAIN +from bookwyrm.utils.db import format_trigger from .book import BookDataModel from . import fields @@ -66,4 +68,31 @@ class Author(BookDataModel): """editions and works both use "book" instead of model_name""" return f"https://{DOMAIN}/author/{self.id}" + class Meta: + """sets up indexes and triggers""" + + triggers = [ + pgtrigger.Trigger( + name="reset_search_vector_on_author_edit", + when=pgtrigger.After, + operation=pgtrigger.UpdateOf("name"), + func=format_trigger( + """WITH book AS ( + SELECT bookwyrm_book.id AS row_id + FROM bookwyrm_author + LEFT OUTER JOIN bookwyrm_book_authors + ON bookwyrm_book_authors.id = new.id + LEFT OUTER JOIN bookwyrm_book + ON bookwyrm_book.id = bookwyrm_book_authors.book_id + ) + UPDATE bookwyrm_book + SET search_vector = '' + FROM book + WHERE id = book.row_id; + RETURN new; + """ + ), + ) + ] + activity_serializer = activitypub.Author diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 4cecc4df6..aaa50c56d 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -104,6 +104,7 @@ INSTALLED_APPS = [ "celery", "django_celery_beat", "imagekit", + "pgtrigger", "storages", ] diff --git a/bookwyrm/utils/db.py b/bookwyrm/utils/db.py new file mode 100644 index 000000000..8b74d9bf5 --- /dev/null +++ b/bookwyrm/utils/db.py @@ -0,0 +1,22 @@ +""" Database utilities """ + +from typing import cast +import sqlparse # type: ignore + + +def format_trigger(sql: str) -> str: + """format SQL trigger before storing + + we remove whitespace and use consistent casing so as to avoid migrations + due to formatting changes. + """ + return cast( + str, + sqlparse.format( + sql, + strip_comments=True, + strip_whitespace=True, + keyword_case="upper", + identifier_case="lower", + ), + ) diff --git a/requirements.txt b/requirements.txt index 36192f148..05fd9d2b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ django-celery-beat==2.4.0 django-compressor==4.3.1 django-imagekit==4.1.0 django-model-utils==4.3.1 +django-pgtrigger==4.10.0 django-sass-processor==1.2.2 django-csp==3.7 environs==9.5.0 From bcb3a343d4514fd9bc2674a918755aaa832c8886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 25 Nov 2023 01:49:14 -0300 Subject: [PATCH 010/123] Fix JOIN in `author_search_vector_trigger`, add missing WHERE clause --- .../0191_migrate_search_vec_triggers_to_pgtriggers.py | 4 ++-- bookwyrm/models/author.py | 3 ++- bookwyrm/tests/test_book_search.py | 2 -- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py index 10354fa67..1e51e529e 100644 --- a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py +++ b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py @@ -32,8 +32,8 @@ class Migration(migrations.Migration): trigger=pgtrigger.compiler.Trigger( name="reset_search_vector_on_author_edit", sql=pgtrigger.compiler.UpsertTriggerSql( - func="WITH book AS (SELECT bookwyrm_book.id AS row_id FROM bookwyrm_author LEFT OUTER JOIN bookwyrm_book_authors ON bookwyrm_book_authors.id = new.id LEFT OUTER JOIN bookwyrm_book ON bookwyrm_book.id = bookwyrm_book_authors.book_id) UPDATE bookwyrm_book SET search_vector = '' FROM book WHERE id = book.row_id;RETURN NEW;", - hash="9c0a472e2bf60e63d593cce49f47972c7b227a00", + func="WITH book AS (SELECT bookwyrm_book.id AS row_id FROM bookwyrm_author LEFT OUTER JOIN bookwyrm_book_authors ON bookwyrm_book_authors.author_id = bookwyrm_author.id LEFT OUTER JOIN bookwyrm_book ON bookwyrm_book.id = bookwyrm_book_authors.book_id WHERE bookwyrm_author.id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM book WHERE id = book.row_id;RETURN NEW;", + hash="abc8ea76fa1bf02a0f56aaae390c1b970bef1278", operation='UPDATE OF "name"', pgid="pgtrigger_reset_search_vector_on_author_edit_a447c", table="bookwyrm_author", diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 9b4f3c1bd..6c5beba50 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -81,9 +81,10 @@ class Author(BookDataModel): SELECT bookwyrm_book.id AS row_id FROM bookwyrm_author LEFT OUTER JOIN bookwyrm_book_authors - ON bookwyrm_book_authors.id = new.id + ON bookwyrm_book_authors.author_id = bookwyrm_author.id LEFT OUTER JOIN bookwyrm_book ON bookwyrm_book.id = bookwyrm_book_authors.book_id + WHERE bookwyrm_author.id = new.id ) UPDATE bookwyrm_book SET search_vector = '' diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index e66ea97be..d2d5b692e 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -1,6 +1,5 @@ """ test searching for books """ import datetime -from unittest import expectedFailure from django.db import connection from django.test import TestCase from django.utils import timezone @@ -207,7 +206,6 @@ class SearchVectorTriggers(TestCase): self.assertEqual(self.edition, self._search_first("Mozilla")) self.assertEqual(self.edition, self._search_first("Name")) - @expectedFailure def test_search_after_updated_author_name(self): """book found under new author name""" self.assertEqual(self.edition, self._search_first("Name")) From 8df408e07eccefb934373f5c69b5068c1bee5046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 25 Nov 2023 16:46:54 -0300 Subject: [PATCH 011/123] Define `search_vector_trigger` via Book.Meta.triggers --- ...grate_search_vec_triggers_to_pgtriggers.py | 20 +++++++++++ bookwyrm/models/book.py | 33 ++++++++++++++++++- bookwyrm/tests/test_book_search.py | 1 - bookwyrm/utils/db.py | 1 + 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py index 1e51e529e..929063781 100644 --- a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py +++ b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py @@ -27,6 +27,20 @@ class Migration(migrations.Migration): ] operations = [ + pgtrigger.migrations.AddTrigger( + model_name="book", + trigger=pgtrigger.compiler.Trigger( + name="update_search_vector_on_book_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="new.search_vector := coalesce(nullif(setweight(to_tsvector('english', coalesce(new.title, '')), 'A'), ''), setweight(to_tsvector('simple', coalesce(new.title, '')), 'A')) || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(bookwyrm_author.name), ' '), '')), 'C') FROM bookwyrm_book LEFT OUTER JOIN bookwyrm_book_authors ON bookwyrm_book.id = bookwyrm_book_authors.book_id LEFT OUTER JOIN bookwyrm_author ON bookwyrm_book_authors.author_id = bookwyrm_author.id WHERE bookwyrm_book.id = new.id ) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D');RETURN NEW;", + hash="9c898d46dfb7492ecd18f6c692bbecfa548f0e85", + operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"', + pgid="pgtrigger_update_search_vector_on_book_edit_bec58", + table="bookwyrm_book", + when="BEFORE", + ), + ), + ), pgtrigger.migrations.AddTrigger( model_name="author", trigger=pgtrigger.compiler.Trigger( @@ -41,6 +55,12 @@ class Migration(migrations.Migration): ), ), ), + migrations.RunSQL( + sql="""DROP TRIGGER IF EXISTS search_vector_trigger ON bookwyrm_book; + DROP FUNCTION IF EXISTS book_trigger; + """, + reverse_sql=search_vector_trigger.sql, + ), migrations.RunSQL( sql="""DROP TRIGGER IF EXISTS author_search_vector_trigger ON bookwyrm_author; DROP FUNCTION IF EXISTS author_trigger; diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 6893b9da1..ed26752e3 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -13,6 +13,7 @@ from django.utils.translation import gettext_lazy as _ from model_utils import FieldTracker from model_utils.managers import InheritanceManager from imagekit.models import ImageSpecField +import pgtrigger from bookwyrm import activitypub from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator @@ -24,6 +25,7 @@ from bookwyrm.settings import ( ENABLE_PREVIEW_IMAGES, ENABLE_THUMBNAIL_GENERATION, ) +from bookwyrm.utils.db import format_trigger from .activitypub_mixin import OrderedCollectionPageMixin, ObjectMixin from .base_model import BookWyrmModel @@ -232,9 +234,38 @@ class Book(BookDataModel): ) class Meta: - """sets up postgres GIN index field""" + """set up indexes and triggers""" + + # pylint: disable=line-too-long indexes = (GinIndex(fields=["search_vector"]),) + triggers = [ + pgtrigger.Trigger( + name="update_search_vector_on_book_edit", + when=pgtrigger.Before, + operation=pgtrigger.Insert + | pgtrigger.UpdateOf("title", "subtitle", "series", "search_vector"), + func=format_trigger( + """new.search_vector := + COALESCE( + NULLIF(setweight(to_tsvector('english', COALESCE(new.title, '')), 'A'), ''), + setweight(to_tsvector('simple', COALESCE(new.title, '')), 'A') + ) || + setweight(to_tsvector('english', COALESCE(new.subtitle, '')), 'B') || + (SELECT setweight(to_tsvector('simple', COALESCE(array_to_string(ARRAY_AGG(bookwyrm_author.name), ' '), '')), 'C') + FROM bookwyrm_book + LEFT OUTER JOIN bookwyrm_book_authors + ON bookwyrm_book.id = bookwyrm_book_authors.book_id + LEFT OUTER JOIN bookwyrm_author + ON bookwyrm_book_authors.author_id = bookwyrm_author.id + WHERE bookwyrm_book.id = new.id + ) || + setweight(to_tsvector('english', COALESCE(new.series, '')), 'D'); + RETURN new; + """ + ), + ) + ] class Work(OrderedCollectionPageMixin, Book): diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index d2d5b692e..f9c75d279 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -213,7 +213,6 @@ class SearchVectorTriggers(TestCase): self.author.name = "Identifier" self.author.save(broadcast=False) - self.edition.refresh_from_db() self.assertFalse(self._search("Name")) self.assertEqual(self.edition, self._search_first("Identifier")) diff --git a/bookwyrm/utils/db.py b/bookwyrm/utils/db.py index 8b74d9bf5..2bb3b9ff6 100644 --- a/bookwyrm/utils/db.py +++ b/bookwyrm/utils/db.py @@ -16,6 +16,7 @@ def format_trigger(sql: str) -> str: sql, strip_comments=True, strip_whitespace=True, + use_space_around_operators=True, keyword_case="upper", identifier_case="lower", ), From 9bcb5b80ea9dd4f11cae26ad1fc752a7cb71dcde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 25 Nov 2023 18:13:05 -0300 Subject: [PATCH 012/123] Further simplify bookwyrm_author trigger --- ..._migrate_search_vec_triggers_to_pgtriggers.py | 4 ++-- bookwyrm/models/author.py | 16 ++++++---------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py index 929063781..ddfe74a8b 100644 --- a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py +++ b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py @@ -46,8 +46,8 @@ class Migration(migrations.Migration): trigger=pgtrigger.compiler.Trigger( name="reset_search_vector_on_author_edit", sql=pgtrigger.compiler.UpsertTriggerSql( - func="WITH book AS (SELECT bookwyrm_book.id AS row_id FROM bookwyrm_author LEFT OUTER JOIN bookwyrm_book_authors ON bookwyrm_book_authors.author_id = bookwyrm_author.id LEFT OUTER JOIN bookwyrm_book ON bookwyrm_book.id = bookwyrm_book_authors.book_id WHERE bookwyrm_author.id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM book WHERE id = book.row_id;RETURN NEW;", - hash="abc8ea76fa1bf02a0f56aaae390c1b970bef1278", + func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;", + hash="e7bbf08711ff3724c58f4d92fb7a082ffb3d7826", operation='UPDATE OF "name"', pgid="pgtrigger_reset_search_vector_on_author_edit_a447c", table="bookwyrm_author", diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 6c5beba50..154b00ccb 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -77,19 +77,15 @@ class Author(BookDataModel): when=pgtrigger.After, operation=pgtrigger.UpdateOf("name"), func=format_trigger( - """WITH book AS ( - SELECT bookwyrm_book.id AS row_id - FROM bookwyrm_author - LEFT OUTER JOIN bookwyrm_book_authors - ON bookwyrm_book_authors.author_id = bookwyrm_author.id - LEFT OUTER JOIN bookwyrm_book - ON bookwyrm_book.id = bookwyrm_book_authors.book_id - WHERE bookwyrm_author.id = new.id + """WITH updated_books AS ( + SELECT book_id + FROM bookwyrm_book_authors + WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' - FROM book - WHERE id = book.row_id; + FROM updated_books + WHERE id = updated_books.book_id; RETURN new; """ ), From bbfbd1e97ab201cd87c6414127c603f615923ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 25 Nov 2023 20:54:09 -0300 Subject: [PATCH 013/123] Add tests for trigger code (i.e. how search_vector is computed) --- bookwyrm/tests/test_book_search.py | 67 +++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index f9c75d279..99b62f0a0 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -137,6 +137,69 @@ class BookSearch(TestCase): self.assertEqual(result.title, "Title") +class SearchVectorTest(TestCase): + """check search_vector is computed correctly""" + + def test_search_vector_simple(self): + """simplest search vector""" + book = self._create_book("Book", "Mary") + self.assertEqual(book.search_vector, "'book':1A 'mary':2C") # A > C (priority) + + def test_search_vector_all_parts(self): + """search vector with subtitle and series""" + # for a book like this we call `to_tsvector("Book Long Mary Bunch")`, hence the + # indexes in the search vector. (priority "D" is the default, and never shown.) + book = self._create_book("Book", "Mary", subtitle="Long", series="Bunch") + self.assertEqual(book.search_vector, "'book':1A 'bunch':4 'long':2B 'mary':3C") + + def test_search_vector_parse_book(self): + """book parts are parsed in english""" + # FIXME: at some point this should stop being the default. + book = self._create_book( + "Edition", "Editor", series="Castle", subtitle="Writing" + ) + self.assertEqual( + book.search_vector, "'castl':4 'edit':1A 'editor':3C 'write':2B" + ) + + def test_search_vector_parse_author(self): + """author name is not stem'd or affected by stop words""" + book = self._create_book("Writing", "Writes") + self.assertEqual(book.search_vector, "'write':1A 'writes':2C") + + book = self._create_book("She Is Writing", "She Writes") + self.assertEqual(book.search_vector, "'she':4C 'write':3A 'writes':5C") + + def test_search_vector_parse_title_empty(self): + """empty parse in English retried as simple title""" + book = self._create_book("Here We", "John") + self.assertEqual(book.search_vector, "'here':1A 'john':3C 'we':2A") + + book = self._create_book("Hear We Come", "John") + self.assertEqual(book.search_vector, "'come':3A 'hear':1A 'john':4C") + + @staticmethod + def _create_book( + title, author_name, /, *, subtitle="", series="", author_alias=None + ): + """quickly create a book""" + work = models.Work.objects.create(title="work") + author = models.Author.objects.create( + name=author_name, aliases=author_alias or [] + ) + edition = models.Edition.objects.create( + title=title, + series=series or None, + subtitle=subtitle or None, + isbn_10="0000000000", + parent_work=work, + ) + edition.authors.add(author) + edition.save(broadcast=False) + edition.refresh_from_db() + return edition + + class SearchVectorTriggers(TestCase): """look for books as they change""" @@ -222,8 +285,8 @@ class SearchVectorTriggers(TestCase): """wrapper around search_title_author""" return self._search(query, return_first=True) - # pylint: disable-next=no-self-use - def _search(self, query, *, return_first=False): + @staticmethod + def _search(query, *, return_first=False): """wrapper around search_title_author""" return book_search.search_title_author( query, min_confidence=0, return_first=return_first From b5805accacb497be62cb4e7bb54a52258a1d12e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 25 Nov 2023 21:47:13 -0300 Subject: [PATCH 014/123] Minor improvements to bookwyrm_book trigger code - do not COALESCE columns that cannot be NULL - do not bring bookwyrm_book to author names JOIN - add comments documenting the four steps --- ...grate_search_vec_triggers_to_pgtriggers.py | 4 ++-- bookwyrm/models/book.py | 21 ++++++++++--------- bookwyrm/tests/test_book_search.py | 16 +++++++++----- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py index ddfe74a8b..5e798b654 100644 --- a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py +++ b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py @@ -32,8 +32,8 @@ class Migration(migrations.Migration): trigger=pgtrigger.compiler.Trigger( name="update_search_vector_on_book_edit", sql=pgtrigger.compiler.UpsertTriggerSql( - func="new.search_vector := coalesce(nullif(setweight(to_tsvector('english', coalesce(new.title, '')), 'A'), ''), setweight(to_tsvector('simple', coalesce(new.title, '')), 'A')) || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(bookwyrm_author.name), ' '), '')), 'C') FROM bookwyrm_book LEFT OUTER JOIN bookwyrm_book_authors ON bookwyrm_book.id = bookwyrm_book_authors.book_id LEFT OUTER JOIN bookwyrm_author ON bookwyrm_book_authors.author_id = bookwyrm_author.id WHERE bookwyrm_book.id = new.id ) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D');RETURN NEW;", - hash="9c898d46dfb7492ecd18f6c692bbecfa548f0e85", + func="new.search_vector := setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(bookwyrm_author.name), ' '), '')), 'C') FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id ) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D');RETURN NEW;", + hash="77d6399497c0a89b0bf09d296e33c396da63705c", operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"', pgid="pgtrigger_update_search_vector_on_book_edit_bec58", table="bookwyrm_book", diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index ed26752e3..e167e2138 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -247,19 +247,20 @@ class Book(BookDataModel): | pgtrigger.UpdateOf("title", "subtitle", "series", "search_vector"), func=format_trigger( """new.search_vector := - COALESCE( - NULLIF(setweight(to_tsvector('english', COALESCE(new.title, '')), 'A'), ''), - setweight(to_tsvector('simple', COALESCE(new.title, '')), 'A') - ) || + -- title, with priority A (parse in English, default to simple if empty) + setweight(COALESCE(nullif( + to_tsvector('english', new.title), ''), + to_tsvector('simple', new.title)), 'A') || + -- subtitle, with priority B (always in English?) setweight(to_tsvector('english', COALESCE(new.subtitle, '')), 'B') || + -- list of authors, with priority C (TODO: add aliases?, bookwyrm-social#3063) (SELECT setweight(to_tsvector('simple', COALESCE(array_to_string(ARRAY_AGG(bookwyrm_author.name), ' '), '')), 'C') - FROM bookwyrm_book - LEFT OUTER JOIN bookwyrm_book_authors - ON bookwyrm_book.id = bookwyrm_book_authors.book_id - LEFT OUTER JOIN bookwyrm_author - ON bookwyrm_book_authors.author_id = bookwyrm_author.id - WHERE bookwyrm_book.id = new.id + FROM bookwyrm_author + LEFT JOIN bookwyrm_book_authors + ON bookwyrm_author.id = bookwyrm_book_authors.author_id + WHERE bookwyrm_book_authors.book_id = new.id ) || + --- last: series name, with lowest priority setweight(to_tsvector('english', COALESCE(new.series, '')), 'D'); RETURN new; """ diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index 99b62f0a0..e9f550a88 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -178,15 +178,17 @@ class SearchVectorTest(TestCase): book = self._create_book("Hear We Come", "John") self.assertEqual(book.search_vector, "'come':3A 'hear':1A 'john':4C") + def test_search_vector_no_author(self): + """book with no authors gets processed normally""" + book = self._create_book("Book", None, series="Bunch") + self.assertEqual(book.search_vector, "'book':1A 'bunch':2") + @staticmethod def _create_book( title, author_name, /, *, subtitle="", series="", author_alias=None ): """quickly create a book""" work = models.Work.objects.create(title="work") - author = models.Author.objects.create( - name=author_name, aliases=author_alias or [] - ) edition = models.Edition.objects.create( title=title, series=series or None, @@ -194,8 +196,12 @@ class SearchVectorTest(TestCase): isbn_10="0000000000", parent_work=work, ) - edition.authors.add(author) - edition.save(broadcast=False) + if author_name is not None: + author = models.Author.objects.create( + name=author_name, aliases=author_alias or [] + ) + edition.authors.add(author) + edition.save(broadcast=False) edition.refresh_from_db() return edition From d6eb390ceed8e6f63926aa8b54a4ef9314b93c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sun, 26 Nov 2023 15:57:51 -0300 Subject: [PATCH 015/123] Add test that forces `book_authors_search_vector_trigger` to execute --- bookwyrm/tests/test_book_search.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index e9f550a88..42e57f773 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -275,6 +275,29 @@ class SearchVectorTriggers(TestCase): self.assertEqual(self.edition, self._search_first("Mozilla")) self.assertEqual(self.edition, self._search_first("Name")) + def test_search_after_author_add_remove_sql(self): + """add/remove author through SQL to ensure execution of book_authors trigger""" + # Tests calling edition.save(), above, pass even if the trigger in + # bookwyrm_book_authors is removed (probably because they trigger the one + # in bookwyrm_book directly). Here we make sure to exercise the former. + new_author = models.Author.objects.create(name="Mozilla") + + with connection.cursor() as cursor: + cursor.execute( + "DELETE FROM bookwyrm_book_authors WHERE book_id = %s", + [self.edition.id], + ) + self.assertFalse(self._search("Name")) + self.assertFalse(self._search("Mozilla")) + + with connection.cursor() as cursor: + cursor.execute( + "INSERT INTO bookwyrm_book_authors (book_id,author_id) VALUES (%s,%s)", + [self.edition.id, new_author.id], + ) + self.assertFalse(self._search("Name")) + self.assertEqual(self.edition, self._search_first("Mozilla")) + def test_search_after_updated_author_name(self): """book found under new author name""" self.assertEqual(self.edition, self._search_first("Name")) From 31a78a5c9efae9559eeb93cc05d18f76869f4ce7 Mon Sep 17 00:00:00 2001 From: Matt Lehrer Date: Thu, 30 Nov 2023 11:12:42 +0100 Subject: [PATCH 016/123] linted --- bookwyrm/tests/views/books/test_editions.py | 26 +++++++++++++++------ bookwyrm/views/books/editions.py | 4 ++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/bookwyrm/tests/views/books/test_editions.py b/bookwyrm/tests/views/books/test_editions.py index bbae517ac..ba380c0bc 100644 --- a/bookwyrm/tests/views/books/test_editions.py +++ b/bookwyrm/tests/views/books/test_editions.py @@ -139,14 +139,17 @@ class BookViews(TestCase): work = models.Work.objects.create(title="test work") edition1 = models.Edition.objects.create(title="first ed", parent_work=work) edition2 = models.Edition.objects.create(title="second ed", parent_work=work) - + models.ReviewRating.objects.create( book=edition1, user=self.local_user, rating=3, ) - self.assertIsInstance(models.ReviewRating.objects.get(user=self.local_user, book=edition1), models.ReviewRating) + self.assertIsInstance( + models.ReviewRating.objects.get(user=self.local_user, book=edition1), + models.ReviewRating, + ) with self.assertRaises(models.ReviewRating.DoesNotExist): models.ReviewRating.objects.get(user=self.local_user, book=edition2) @@ -154,7 +157,10 @@ class BookViews(TestCase): request.user = self.local_user views.switch_edition(request) - self.assertIsInstance(models.ReviewRating.objects.get(user=self.local_user, book=edition2), models.ReviewRating) + self.assertIsInstance( + models.ReviewRating.objects.get(user=self.local_user, book=edition2), + models.ReviewRating, + ) with self.assertRaises(models.ReviewRating.DoesNotExist): models.ReviewRating.objects.get(user=self.local_user, book=edition1) @@ -167,16 +173,19 @@ class BookViews(TestCase): work = models.Work.objects.create(title="test work") edition1 = models.Edition.objects.create(title="first ed", parent_work=work) edition2 = models.Edition.objects.create(title="second ed", parent_work=work) - + models.Review.objects.create( book=edition1, user=self.local_user, name="blah", rating=3, - content="not bad" + content="not bad", ) - self.assertIsInstance(models.Review.objects.get(user=self.local_user, book=edition1), models.Review) + self.assertIsInstance( + models.Review.objects.get(user=self.local_user, book=edition1), + models.Review, + ) with self.assertRaises(models.Review.DoesNotExist): models.Review.objects.get(user=self.local_user, book=edition2) @@ -184,6 +193,9 @@ class BookViews(TestCase): request.user = self.local_user views.switch_edition(request) - self.assertIsInstance(models.Review.objects.get(user=self.local_user, book=edition2), models.Review) + self.assertIsInstance( + models.Review.objects.get(user=self.local_user, book=edition2), + models.Review, + ) with self.assertRaises(models.Review.DoesNotExist): models.Review.objects.get(user=self.local_user, book=edition1) diff --git a/bookwyrm/views/books/editions.py b/bookwyrm/views/books/editions.py index 2bbb1c579..572dc9786 100644 --- a/bookwyrm/views/books/editions.py +++ b/bookwyrm/views/books/editions.py @@ -107,9 +107,9 @@ def switch_edition(request): book__parent_work=new_edition.parent_work, user=request.user ) for review in reviews.all(): - # because ratings are a subclass of reviews, + # because ratings are a subclass of reviews, # this will pick up both ratings and reviews review.book = new_edition - review.save() + review.save() return redirect(f"/book/{new_edition.id}") From d6f7f76c4d8c35ebafb994f6f4ee01b932ae2bf0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 11:37:01 -0800 Subject: [PATCH 017/123] Removes outdated/unused version and updating code I had the bright idea of creating this update script but it doesn't work and hasn't been maintained, so it's just sitting there causing confusing and requiring weird things to exist in other places. Now, the unused `version` field can be removed and I can scrap the management command for getting versions. --- .../management/commands/instance_version.py | 54 ------------------- ..._version_sitesettings_available_version.py | 18 +++++++ bookwyrm/models/site.py | 2 +- bookwyrm/views/admin/dashboard.py | 18 ++----- update.sh | 37 ------------- 5 files changed, 24 insertions(+), 105 deletions(-) delete mode 100644 bookwyrm/management/commands/instance_version.py create mode 100644 bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py delete mode 100755 update.sh diff --git a/bookwyrm/management/commands/instance_version.py b/bookwyrm/management/commands/instance_version.py deleted file mode 100644 index ca150d640..000000000 --- a/bookwyrm/management/commands/instance_version.py +++ /dev/null @@ -1,54 +0,0 @@ -""" Get your admin code to allow install """ -from django.core.management.base import BaseCommand - -from bookwyrm import models -from bookwyrm.settings import VERSION - - -# pylint: disable=no-self-use -class Command(BaseCommand): - """command-line options""" - - help = "What version is this?" - - def add_arguments(self, parser): - """specify which function to run""" - parser.add_argument( - "--current", - action="store_true", - help="Version stored in database", - ) - parser.add_argument( - "--target", - action="store_true", - help="Version stored in settings", - ) - parser.add_argument( - "--update", - action="store_true", - help="Update database version", - ) - - # pylint: disable=unused-argument - def handle(self, *args, **options): - """execute init""" - site = models.SiteSettings.objects.get() - current = site.version or "0.0.1" - target = VERSION - if options.get("current"): - print(current) - return - - if options.get("target"): - print(target) - return - - if options.get("update"): - site.version = target - site.save() - return - - if current != target: - print(f"{current}/{target}") - else: - print(current) diff --git a/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py b/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py new file mode 100644 index 000000000..219ae32f6 --- /dev/null +++ b/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-01-02 19:36 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0191_merge_20240102_0326'), + ] + + operations = [ + migrations.RenameField( + model_name='sitesettings', + old_name='version', + new_name='available_version', + ), + ] diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index bd53f1f07..7ca7e0015 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -45,7 +45,7 @@ class SiteSettings(SiteModel): default_theme = models.ForeignKey( "Theme", null=True, blank=True, on_delete=models.SET_NULL ) - version = models.CharField(null=True, blank=True, max_length=10) + available_version = models.CharField(null=True, blank=True, max_length=10) # admin setup options install_mode = models.BooleanField(default=False) diff --git a/bookwyrm/views/admin/dashboard.py b/bookwyrm/views/admin/dashboard.py index 9d256fc6c..c5648ff11 100644 --- a/bookwyrm/views/admin/dashboard.py +++ b/bookwyrm/views/admin/dashboard.py @@ -15,7 +15,6 @@ from django.views import View from csp.decorators import csp_update from bookwyrm import models, settings -from bookwyrm.connectors.abstract_connector import get_data from bookwyrm.utils import regex @@ -59,18 +58,11 @@ class Dashboard(View): == site._meta.get_field("privacy_policy").get_default() ) - # check version - - try: - release = get_data(settings.RELEASE_API, timeout=3) - available_version = release.get("tag_name", None) - if available_version and version.parse(available_version) > version.parse( - settings.VERSION - ): - data["current_version"] = settings.VERSION - data["available_version"] = available_version - except: # pylint: disable= bare-except - pass + if site.available_version and version.parse(site.available_version) > version.parse( + settings.VERSION + ): + data["current_version"] = settings.VERSION + data["available_version"] = site.available_version return TemplateResponse(request, "settings/dashboard/dashboard.html", data) diff --git a/update.sh b/update.sh deleted file mode 100755 index 727ce1b24..000000000 --- a/update.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -e - -# determine inital and target versions -initial_version="`./bw-dev runweb python manage.py instance_version --current`" -target_version="`./bw-dev runweb python manage.py instance_version --target`" - -initial_version="`echo $initial_version | tail -n 1 | xargs`" -target_version="`echo $target_version | tail -n 1 | xargs`" -if [[ "$initial_version" = "$target_version" ]]; then - echo "Already up to date; version $initial_version" - exit -fi - -echo "---------------------------------------" -echo "Updating from version: $initial_version" -echo ".......... to version: $target_version" -echo "---------------------------------------" - -function version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } - -# execute scripts between initial and target -for version in `ls -A updates/ | sort -V `; do - if version_gt $initial_version $version; then - # too early - continue - fi - if version_gt $version $target_version; then - # too late - continue - fi - echo "Running tasks for version $version" - ./updates/$version -done - -./bw-dev runweb python manage.py instance_version --update -echo "✨ ----------- Done! --------------- ✨" From 5509941aa4d14fc5422644cac5e8ff658e1fbd5b Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 12:23:32 -0800 Subject: [PATCH 018/123] Adds schedule-able task to check for version updates --- ..._version_sitesettings_available_version.py | 8 +++---- bookwyrm/models/site.py | 14 ++++++++++++ .../settings/dashboard/dashboard.html | 4 ++++ .../dashboard/warnings/check_for_updates.html | 22 +++++++++++++++++++ bookwyrm/views/admin/dashboard.py | 21 +++++++++++++++++- 5 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html diff --git a/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py b/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py index 219ae32f6..db67b4e92 100644 --- a/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py +++ b/bookwyrm/migrations/0192_rename_version_sitesettings_available_version.py @@ -6,13 +6,13 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('bookwyrm', '0191_merge_20240102_0326'), + ("bookwyrm", "0191_merge_20240102_0326"), ] operations = [ migrations.RenameField( - model_name='sitesettings', - old_name='version', - new_name='available_version', + model_name="sitesettings", + old_name="version", + new_name="available_version", ), ] diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index 7ca7e0015..f82a4d94b 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -10,8 +10,11 @@ from django.dispatch import receiver from django.utils import timezone from model_utils import FieldTracker +from bookwyrm.connectors.abstract_connector import get_data from bookwyrm.preview_images import generate_site_preview_image_task from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, STATIC_FULL_URL +from bookwyrm.settings import RELEASE_API +from bookwyrm.tasks import app, MISC from .base_model import BookWyrmModel, new_access_code from .user import User from .fields import get_absolute_url @@ -244,3 +247,14 @@ def preview_image(instance, *args, **kwargs): if len(changed_fields) > 0: generate_site_preview_image_task.delay() + + +@app.task(queue=MISC) +def check_for_updates_task(): + """ See if git remote knows about a new version """ + site = SiteSettings.objects.get() + release = get_data(RELEASE_API, timeout=3) + available_version = release.get("tag_name", None) + if available_version: + site.available_version = available_version + site.save(update_fields=["available_version"]) diff --git a/bookwyrm/templates/settings/dashboard/dashboard.html b/bookwyrm/templates/settings/dashboard/dashboard.html index 4c109c7e1..d43b3bade 100644 --- a/bookwyrm/templates/settings/dashboard/dashboard.html +++ b/bookwyrm/templates/settings/dashboard/dashboard.html @@ -45,6 +45,10 @@ {% include 'settings/dashboard/warnings/update_version.html' with warning_level="warning" fullwidth=True %} {% endif %} + {% if schedule_form %} + {% include 'settings/dashboard/warnings/check_for_updates.html' with warning_level="success" fullwidth=True %} + {% endif %} + {% if missing_privacy or missing_conduct %}
{% if missing_privacy %} diff --git a/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html b/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html new file mode 100644 index 000000000..07f11a62d --- /dev/null +++ b/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html @@ -0,0 +1,22 @@ +{% extends 'settings/dashboard/warnings/layout.html' %} +{% load i18n %} + +{% block warning_text %} + +
+ {% csrf_token %} + +

+ {% blocktrans trimmed with current=current_version available=available_version %} + Check for available version updates? (Recommended) + {% endblocktrans %} +

+ + {{ schedule_form.every.as_hidden }} + {{ schedule_form.period.as_hidden }} + + +
+ +{% endblock %} + diff --git a/bookwyrm/views/admin/dashboard.py b/bookwyrm/views/admin/dashboard.py index c5648ff11..ea0675f59 100644 --- a/bookwyrm/views/admin/dashboard.py +++ b/bookwyrm/views/admin/dashboard.py @@ -6,15 +6,18 @@ from dateutil.parser import parse from packaging import version from django.contrib.auth.decorators import login_required, permission_required +from django.db import transaction from django.db.models import Q +from django.shortcuts import redirect from django.template.response import TemplateResponse from django.utils import timezone from django.utils.decorators import method_decorator from django.views import View +from django_celery_beat.models import PeriodicTask from csp.decorators import csp_update -from bookwyrm import models, settings +from bookwyrm import forms, models, settings from bookwyrm.utils import regex @@ -64,8 +67,24 @@ class Dashboard(View): data["current_version"] = settings.VERSION data["available_version"] = site.available_version + if not PeriodicTask.objects.filter(name="check-for-updates").exists(): + data["schedule_form"] = forms.IntervalScheduleForm({"every": 1, "period": "days"}) + return TemplateResponse(request, "settings/dashboard/dashboard.html", data) + def post(self, request): + """ Create a schedule task to check for updates """ + schedule_form = forms.IntervalScheduleForm(request.POST) + + with transaction.atomic(): + schedule = schedule_form.save(request) + PeriodicTask.objects.get_or_create( + interval=schedule, + name="check-for-updates", + task="bookwyrm.models.site.check_for_updates_task" + ) + return redirect("settings-dashboard") + def get_charts_and_stats(request): """Defines the dashboard charts""" From f36af42f414196f3e12d30a843470979cbcb9713 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 13:05:44 -0800 Subject: [PATCH 019/123] Adds view to see scheduled tasks --- bookwyrm/models/site.py | 2 +- bookwyrm/templates/settings/layout.html | 4 + bookwyrm/templates/settings/schedules.html | 116 +++++++++++++++++++++ bookwyrm/urls.py | 5 + bookwyrm/views/__init__.py | 1 + bookwyrm/views/admin/dashboard.py | 14 +-- bookwyrm/views/admin/schedule.py | 23 ++++ 7 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 bookwyrm/templates/settings/schedules.html create mode 100644 bookwyrm/views/admin/schedule.py diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index f82a4d94b..ad0dbff64 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -251,7 +251,7 @@ def preview_image(instance, *args, **kwargs): @app.task(queue=MISC) def check_for_updates_task(): - """ See if git remote knows about a new version """ + """See if git remote knows about a new version""" site = SiteSettings.objects.get() release = get_data(RELEASE_API, timeout=3) available_version = release.get("tag_name", None) diff --git a/bookwyrm/templates/settings/layout.html b/bookwyrm/templates/settings/layout.html index dcaaaeb38..70c7ef0f4 100644 --- a/bookwyrm/templates/settings/layout.html +++ b/bookwyrm/templates/settings/layout.html @@ -85,6 +85,10 @@ {% url 'settings-celery' as url %} {% trans "Celery status" %} +
  • + {% url 'settings-schedules' as url %} + {% trans "Scheduled tasks" %} +
  • {% url 'settings-email-config' as url %} {% trans "Email Configuration" %} diff --git a/bookwyrm/templates/settings/schedules.html b/bookwyrm/templates/settings/schedules.html new file mode 100644 index 000000000..fe096092d --- /dev/null +++ b/bookwyrm/templates/settings/schedules.html @@ -0,0 +1,116 @@ +{% extends 'settings/layout.html' %} +{% load i18n %} +{% load humanize %} +{% load utilities %} + +{% block title %} +{% trans "Scheduled tasks" %} +{% endblock %} + +{% block header %} +{% trans "Scheduled tasks" %} +{% endblock %} + +{% block panel %} + +
    +

    {% trans "Tasks" %}

    +
    + + + + + + + + + + + {% for task in tasks %} + + + + + + + + + + {% empty %} + + + + {% endfor %} +
    + {% trans "Name" %} + + {% trans "Celery task" %} + + {% trans "Date changed" %} + + {% trans "Last run at" %} + + {% trans "Schedule" %} + + {% trans "Schedule ID" %} + + {% trans "Enabled" %} +
    + {{ task.name }} + + {{ task.task }} + + {{ task.date_changed }} + + {{ task.last_run_at }} + + {% firstof task.interval task.crontab "None" %} + + {{ task.interval.id }} + + {{ task.enabled|yesno }} +
    + {% trans "No scheduled tasks" %} +
    +
    +
    + +
    +

    {% trans "Schedules" %}

    +
    + + + + + + + {% for schedule in schedules %} + + + + + + {% empty %} + + + + {% endfor %} +
    + {% trans "ID" %} + + {% trans "Schedule" %} + + {% trans "Tasks" %} +
    + {{ schedule.id }} + + {{ schedule }} + + {{ schedule.periodictask_set.count }} +
    + {% trans "No schedules found" %} +
    +
    +
    + +{% endblock %} diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 76e60245b..64742347a 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -359,6 +359,11 @@ urlpatterns = [ re_path( r"^settings/celery/ping/?$", views.celery_ping, name="settings-celery-ping" ), + re_path( + r"^settings/schedules/?$", + views.ScheduledTasks.as_view(), + name="settings-schedules", + ), re_path( r"^settings/email-config/?$", views.EmailConfig.as_view(), diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 3be813208..d77f2675f 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -5,6 +5,7 @@ from .admin.announcements import EditAnnouncement, delete_announcement from .admin.automod import AutoMod, automod_delete, run_automod from .admin.automod import schedule_automod_task, unschedule_automod_task from .admin.celery_status import CeleryStatus, celery_ping +from .admin.schedule import ScheduledTasks from .admin.dashboard import Dashboard from .admin.federation import Federation, FederatedServer from .admin.federation import AddFederatedServer, ImportServerBlocklist diff --git a/bookwyrm/views/admin/dashboard.py b/bookwyrm/views/admin/dashboard.py index ea0675f59..4b2575fa6 100644 --- a/bookwyrm/views/admin/dashboard.py +++ b/bookwyrm/views/admin/dashboard.py @@ -61,19 +61,21 @@ class Dashboard(View): == site._meta.get_field("privacy_policy").get_default() ) - if site.available_version and version.parse(site.available_version) > version.parse( - settings.VERSION - ): + if site.available_version and version.parse( + site.available_version + ) > version.parse(settings.VERSION): data["current_version"] = settings.VERSION data["available_version"] = site.available_version if not PeriodicTask.objects.filter(name="check-for-updates").exists(): - data["schedule_form"] = forms.IntervalScheduleForm({"every": 1, "period": "days"}) + data["schedule_form"] = forms.IntervalScheduleForm( + {"every": 1, "period": "days"} + ) return TemplateResponse(request, "settings/dashboard/dashboard.html", data) def post(self, request): - """ Create a schedule task to check for updates """ + """Create a schedule task to check for updates""" schedule_form = forms.IntervalScheduleForm(request.POST) with transaction.atomic(): @@ -81,7 +83,7 @@ class Dashboard(View): PeriodicTask.objects.get_or_create( interval=schedule, name="check-for-updates", - task="bookwyrm.models.site.check_for_updates_task" + task="bookwyrm.models.site.check_for_updates_task", ) return redirect("settings-dashboard") diff --git a/bookwyrm/views/admin/schedule.py b/bookwyrm/views/admin/schedule.py new file mode 100644 index 000000000..ce5944ee5 --- /dev/null +++ b/bookwyrm/views/admin/schedule.py @@ -0,0 +1,23 @@ +""" Scheduled celery tasks """ +from django.contrib.auth.decorators import login_required, permission_required +from django.template.response import TemplateResponse +from django.utils.decorators import method_decorator +from django.views import View +from django_celery_beat.models import PeriodicTask, IntervalSchedule + + +@method_decorator(login_required, name="dispatch") +@method_decorator( + permission_required("bookwyrm.edit_instance_settings", raise_exception=True), + name="dispatch", +) +# pylint: disable=no-self-use +class ScheduledTasks(View): + """Manage automated flagging""" + + def get(self, request): + """view schedules""" + data = {} + data["tasks"] = PeriodicTask.objects.all() + data["schedules"] = IntervalSchedule.objects.all() + return TemplateResponse(request, "settings/schedules.html", data) From 8be9e91d2162871faae0aaabbecdf8da449b6c2d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 13:16:53 -0800 Subject: [PATCH 020/123] Re-use schedules rather than creating new ones --- bookwyrm/views/admin/automod.py | 4 ++-- bookwyrm/views/admin/dashboard.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bookwyrm/views/admin/automod.py b/bookwyrm/views/admin/automod.py index 9a32dd9ee..58818ad9b 100644 --- a/bookwyrm/views/admin/automod.py +++ b/bookwyrm/views/admin/automod.py @@ -6,7 +6,7 @@ from django.template.response import TemplateResponse from django.utils.decorators import method_decorator from django.views import View from django.views.decorators.http import require_POST -from django_celery_beat.models import PeriodicTask +from django_celery_beat.models import PeriodicTask, IntervalSchedule from bookwyrm import forms, models @@ -54,7 +54,7 @@ def schedule_automod_task(request): return TemplateResponse(request, "settings/automod/rules.html", data) with transaction.atomic(): - schedule = form.save(request) + schedule, _ = IntervalSchedule.objects.get_or_create(**form.cleaned_data) PeriodicTask.objects.get_or_create( interval=schedule, name="automod-task", diff --git a/bookwyrm/views/admin/dashboard.py b/bookwyrm/views/admin/dashboard.py index 4b2575fa6..a4c630067 100644 --- a/bookwyrm/views/admin/dashboard.py +++ b/bookwyrm/views/admin/dashboard.py @@ -13,7 +13,7 @@ from django.template.response import TemplateResponse from django.utils import timezone from django.utils.decorators import method_decorator from django.views import View -from django_celery_beat.models import PeriodicTask +from django_celery_beat.models import PeriodicTask, IntervalSchedule from csp.decorators import csp_update @@ -79,7 +79,9 @@ class Dashboard(View): schedule_form = forms.IntervalScheduleForm(request.POST) with transaction.atomic(): - schedule = schedule_form.save(request) + schedule, _ = IntervalSchedule.objects.get_or_create( + **schedule_form.cleaned_data + ) PeriodicTask.objects.get_or_create( interval=schedule, name="check-for-updates", From 193a1c7d54564c3ab35be20e27681566c2305581 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 13:28:25 -0800 Subject: [PATCH 021/123] updates wording and fixes get or create logic --- VERSION | 2 +- .../settings/dashboard/warnings/check_for_updates.html | 4 ++-- bookwyrm/views/admin/dashboard.py | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 39e898a4f..ee6cdce3c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.1 +0.6.1 diff --git a/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html b/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html index 07f11a62d..f0a2a8013 100644 --- a/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html +++ b/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html @@ -8,14 +8,14 @@

    {% blocktrans trimmed with current=current_version available=available_version %} - Check for available version updates? (Recommended) + Would you like to automatically check for new BookWyrm releases? (recommended) {% endblocktrans %}

    {{ schedule_form.every.as_hidden }} {{ schedule_form.period.as_hidden }} - + {% endblock %} diff --git a/bookwyrm/views/admin/dashboard.py b/bookwyrm/views/admin/dashboard.py index a4c630067..21b19bf16 100644 --- a/bookwyrm/views/admin/dashboard.py +++ b/bookwyrm/views/admin/dashboard.py @@ -77,6 +77,8 @@ class Dashboard(View): def post(self, request): """Create a schedule task to check for updates""" schedule_form = forms.IntervalScheduleForm(request.POST) + if not schedule_form.is_valid(): + raise schedule_form.ValidationError(schedule_form.errors) with transaction.atomic(): schedule, _ = IntervalSchedule.objects.get_or_create( From d287581620a5d1238988803d05ce2ce9c6cbf52b Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 13:31:18 -0800 Subject: [PATCH 022/123] Fixes html validation error --- .../settings/dashboard/warnings/check_for_updates.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html b/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html index f0a2a8013..00f320824 100644 --- a/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html +++ b/bookwyrm/templates/settings/dashboard/warnings/check_for_updates.html @@ -1,6 +1,8 @@ {% extends 'settings/dashboard/warnings/layout.html' %} {% load i18n %} +{% block warning_link %}#{% endblock %} + {% block warning_text %}
    From 6cd2c9113506cf71756fc1bb5f025297fbcc4f53 Mon Sep 17 00:00:00 2001 From: Wesley Aptekar-Cassels Date: Thu, 4 Jan 2024 18:58:12 -0500 Subject: [PATCH 023/123] Allow page numbers to be text, instead of integers. Fixes: #2640 --- .../0192_make_page_positions_text.py | 23 +++++++++++++++++++ bookwyrm/models/status.py | 10 ++++---- .../snippets/create_status/quotation.html | 6 ++--- bookwyrm/tests/views/books/test_book.py | 8 ++++--- 4 files changed, 36 insertions(+), 11 deletions(-) create mode 100644 bookwyrm/migrations/0192_make_page_positions_text.py diff --git a/bookwyrm/migrations/0192_make_page_positions_text.py b/bookwyrm/migrations/0192_make_page_positions_text.py new file mode 100644 index 000000000..940a9e941 --- /dev/null +++ b/bookwyrm/migrations/0192_make_page_positions_text.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.23 on 2024-01-04 23:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0191_merge_20240102_0326"), + ] + + operations = [ + migrations.AlterField( + model_name="quotation", + name="endposition", + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name="quotation", + name="position", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index cc44fe2bf..f33c32824 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -338,11 +338,13 @@ class Quotation(BookStatus): quote = fields.HtmlField() raw_quote = models.TextField(blank=True, null=True) - position = models.IntegerField( - validators=[MinValueValidator(0)], null=True, blank=True + position = models.TextField( + null=True, + blank=True, ) - endposition = models.IntegerField( - validators=[MinValueValidator(0)], null=True, blank=True + endposition = models.TextField( + null=True, + blank=True, ) position_mode = models.CharField( max_length=3, diff --git a/bookwyrm/templates/snippets/create_status/quotation.html b/bookwyrm/templates/snippets/create_status/quotation.html index bd1d817ad..dc17585a9 100644 --- a/bookwyrm/templates/snippets/create_status/quotation.html +++ b/bookwyrm/templates/snippets/create_status/quotation.html @@ -56,8 +56,7 @@ uuid: a unique identifier used to make html "id" attributes unique and clarify j Date: Tue, 9 Jan 2024 15:31:05 +0530 Subject: [PATCH 024/123] Issue-3187: changes --- bookwyrm/templates/feed/feed.html | 2 +- bookwyrm/views/feed.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bookwyrm/templates/feed/feed.html b/bookwyrm/templates/feed/feed.html index 7ecf10b70..820314b7a 100644 --- a/bookwyrm/templates/feed/feed.html +++ b/bookwyrm/templates/feed/feed.html @@ -33,7 +33,7 @@ - {% if request.user.show_goal and not goal and tab.key == 'home' %} + {% if request.user.show_goal and not goal and tab.key == 'home' and has_read_throughs %} {% now 'Y' as year %}
    {% include 'feed/goal_card.html' with year=year %} diff --git a/bookwyrm/views/feed.py b/bookwyrm/views/feed.py index 17218b93e..381d233e9 100644 --- a/bookwyrm/views/feed.py +++ b/bookwyrm/views/feed.py @@ -52,6 +52,8 @@ class Feed(View): suggestions = suggested_users.get_suggestions(request.user) + readthroughs = models.ReadThrough.objects.filter(user=request.user) + data = { **feed_page_data(request.user), **{ @@ -66,6 +68,7 @@ class Feed(View): "path": f"/{tab['key']}", "annual_summary_year": get_annual_summary_year(), "has_tour": True, + "has_read_throughs": True if len(readthroughs) else False, }, } return TemplateResponse(request, "feed/feed.html", data) From 9a487b0442c0d9e94b511491bf61bd00b3cfebfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 13 Jan 2024 17:51:14 +0100 Subject: [PATCH 025/123] Ensure dev-tools uses bookworm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 1937177e1 ("dev-tools: use apt source for Node instead of setup script"), I introduced the use of `Signed-By` with a public key block, which is only supported in bookworm (bullseye only supports fingerprints, TTBOMK). Python's Docker images already use bookworm by default, but we explicitly require it now to avoid build errors if someone has a very old image laying around (see, e.g., #3190). (This can be dropped after Debian 13 ‘trixie’ is released.) --- dev-tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/Dockerfile b/dev-tools/Dockerfile index 3b7740a78..6c132944f 100644 --- a/dev-tools/Dockerfile +++ b/dev-tools/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.9-bookworm WORKDIR /app/dev-tools ENV PATH="/app/dev-tools/node_modules/.bin:$PATH" From eb13eb98828509b96c34fbbcae272d37050789f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 13 Jan 2024 18:55:13 +0100 Subject: [PATCH 026/123] Invalidate `active_shelf` when switching editions --- bookwyrm/views/books/editions.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bookwyrm/views/books/editions.py b/bookwyrm/views/books/editions.py index 54d1bd84c..0d59d0bcc 100644 --- a/bookwyrm/views/books/editions.py +++ b/bookwyrm/views/books/editions.py @@ -3,6 +3,7 @@ from functools import reduce import operator from django.contrib.auth.decorators import login_required +from django.core.cache import cache as django_cache from django.core.paginator import Paginator from django.db import transaction from django.db.models import Q @@ -103,4 +104,10 @@ def switch_edition(request): readthrough.book = new_edition readthrough.save() + django_cache.delete_many( + [ + f"active_shelf-{request.user.id}-{book_id}" + for book_id in new_edition.parent_work.editions.values_list("id", flat=True) + ] + ) return redirect(f"/book/{new_edition.id}") From 5ef104b802dce740835f4a34bb4403459e2791bc Mon Sep 17 00:00:00 2001 From: Rohan Sureshkumar Date: Mon, 15 Jan 2024 17:22:33 +0530 Subject: [PATCH 027/123] Issue-3187: addressing review comments --- bookwyrm/templates/feed/feed.html | 4 ++-- bookwyrm/views/feed.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bookwyrm/templates/feed/feed.html b/bookwyrm/templates/feed/feed.html index 820314b7a..56c380202 100644 --- a/bookwyrm/templates/feed/feed.html +++ b/bookwyrm/templates/feed/feed.html @@ -33,7 +33,7 @@ - {% if request.user.show_goal and not goal and tab.key == 'home' and has_read_throughs %} + {% if request.user.show_goal and not goal and tab.key == 'home' %} {% now 'Y' as year %}
    {% include 'feed/goal_card.html' with year=year %} @@ -41,7 +41,7 @@
    {% endif %} - {% if annual_summary_year and tab.key == 'home' %} + {% if annual_summary_year and tab.key == 'home' and has_read_throughs %}
    • -
    +
    {% if superlatives.top_rated %} {% with book=superlatives.top_rated.default_edition rating=superlatives.top_rated.rating %} -
    +
    @@ -53,7 +53,7 @@ {% if superlatives.wanted %} {% with book=superlatives.wanted.default_edition %} -
    +
    @@ -72,7 +72,7 @@ {% if superlatives.controversial %} {% with book=superlatives.controversial.default_edition %} -
    +
    From 8e2649ba3b26bc03ff90d8a5790f7b19301ebd34 Mon Sep 17 00:00:00 2001 From: Rohan Sureshkumar Date: Thu, 18 Jan 2024 21:23:25 +0530 Subject: [PATCH 029/123] Issue-3187: change variable name and code formatting --- bookwyrm/templates/feed/feed.html | 2 +- bookwyrm/views/feed.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bookwyrm/templates/feed/feed.html b/bookwyrm/templates/feed/feed.html index 56c380202..1b6cf29ff 100644 --- a/bookwyrm/templates/feed/feed.html +++ b/bookwyrm/templates/feed/feed.html @@ -41,7 +41,7 @@ {% endif %} - {% if annual_summary_year and tab.key == 'home' and has_read_throughs %} + {% if annual_summary_year and tab.key == 'home' and has_summary_read_throughs %}
    -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/bookwyrm/views/feed.py b/bookwyrm/views/feed.py index 17218b93e..2d91990d0 100644 --- a/bookwyrm/views/feed.py +++ b/bookwyrm/views/feed.py @@ -197,6 +197,8 @@ class Status(View): "status": status, "children": children, "ancestors": ancestors, + "title": status.page_title, + "description": status.page_description, "preview": preview, }, } From ad56024ffe8adac7a8cab916b160b2f18edd2f1d Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 20 Jan 2024 17:18:50 +0100 Subject: [PATCH 036/123] Add Status.page_image property --- bookwyrm/models/status.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 04fb8daa3..236826a2b 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -190,6 +190,15 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): """description of the page in meta tags when only this status is shown""" return None + @property + def page_image(self): + """image to use as preview in meta tags when only this status is shown""" + if self.mention_books.exists(): + book = self.mention_books.first() + return book.preview_image + else: + return self.user.preview_image + def to_replies(self, **kwargs): """helper function for loading AP serialized replies to a status""" return self.to_ordered_collection( @@ -313,6 +322,10 @@ class BookStatus(Status): abstract = True + @property + def page_image(self): + return self.book.preview_image or super().page_image + class Comment(BookStatus): """like a review but without a rating and transient""" From 290ee997b3c935297811f033d8da0a564ba48f52 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 20 Jan 2024 17:24:20 +0100 Subject: [PATCH 037/123] Refactor OpenGraph tags logic --- bookwyrm/templates/snippets/opengraph.html | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/bookwyrm/templates/snippets/opengraph.html b/bookwyrm/templates/snippets/opengraph.html index 1e87a464f..78a6b1b3f 100644 --- a/bookwyrm/templates/snippets/opengraph.html +++ b/bookwyrm/templates/snippets/opengraph.html @@ -1,24 +1,25 @@ {% load static %} -{% if preview_images_enabled is True %} +{% firstof image site.preview_image as page_image %} +{% if page_image %} - {% if image %} - - - {% else %} - - - {% endif %} + + +{% elif site.logo %} + + + + {% else %} - - + + + {% endif %} - - - - +{% firstof description site.instance_tagline as description %} + + From ea9d3f8ba1ac3db3a050b0355bd3de3d20cd061e Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 20 Jan 2024 17:25:20 +0100 Subject: [PATCH 038/123] Use Status.page_image for OpenGraph tags --- bookwyrm/templates/feed/status.html | 4 ++-- bookwyrm/views/feed.py | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/bookwyrm/templates/feed/status.html b/bookwyrm/templates/feed/status.html index b381c3714..64e992a01 100644 --- a/bookwyrm/templates/feed/status.html +++ b/bookwyrm/templates/feed/status.html @@ -6,7 +6,7 @@ {% block opengraph %} - {% include 'snippets/opengraph.html' with image=preview %} + {% include 'snippets/opengraph.html' with image=page_image %} {% endblock %} @@ -39,4 +39,4 @@
    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/bookwyrm/views/feed.py b/bookwyrm/views/feed.py index 2d91990d0..d1feb278e 100644 --- a/bookwyrm/views/feed.py +++ b/bookwyrm/views/feed.py @@ -185,12 +185,6 @@ class Status(View): params=[status.id, visible_thread, visible_thread], ) - preview = None - if hasattr(status, "book"): - preview = status.book.preview_image - elif status.mention_books.exists(): - preview = status.mention_books.first().preview_image - data = { **feed_page_data(request.user), **{ @@ -199,7 +193,7 @@ class Status(View): "ancestors": ancestors, "title": status.page_title, "description": status.page_description, - "preview": preview, + "page_image": status.page_image, }, } return TemplateResponse(request, "feed/status.html", data) From 646b27b7a7e1099e81d5c3fa4c2b0008f54c889e Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 20 Jan 2024 17:28:51 +0100 Subject: [PATCH 039/123] OpenGraph: fall back on book cover when preview images are disabled --- bookwyrm/models/status.py | 4 ++-- bookwyrm/templates/book/book.html | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 236826a2b..94893d6ae 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -195,7 +195,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): """image to use as preview in meta tags when only this status is shown""" if self.mention_books.exists(): book = self.mention_books.first() - return book.preview_image + return book.preview_image or book.cover else: return self.user.preview_image @@ -324,7 +324,7 @@ class BookStatus(Status): @property def page_image(self): - return self.book.preview_image or super().page_image + return self.book.preview_image or self.book.cover or super().page_image class Comment(BookStatus): diff --git a/bookwyrm/templates/book/book.html b/bookwyrm/templates/book/book.html index 8e76fb014..4c345832e 100644 --- a/bookwyrm/templates/book/book.html +++ b/bookwyrm/templates/book/book.html @@ -9,7 +9,8 @@ {% block title %}{{ book|book_title }}{% endblock %} {% block opengraph %} - {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book.preview_image %} + {% firstof book.preview_image book.cover as book_image %} + {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book_image %} {% endblock %} {% block content %} From eb6bea013fd1634f178da689e4843adcd05a6296 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sun, 21 Jan 2024 11:04:08 +0100 Subject: [PATCH 040/123] Fix pylint warning --- bookwyrm/models/status.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 94893d6ae..0c9b18cc9 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -196,8 +196,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): if self.mention_books.exists(): book = self.mention_books.first() return book.preview_image or book.cover - else: - return self.user.preview_image + return self.user.preview_image def to_replies(self, **kwargs): """helper function for loading AP serialized replies to a status""" From 30ba8d37dc130ac547c59a5b7c2b7fae3d529214 Mon Sep 17 00:00:00 2001 From: Wesley Aptekar-Cassels Date: Tue, 23 Jan 2024 18:19:31 -0500 Subject: [PATCH 041/123] Add redis automatic rewrite configuration. This should hopefully prevent the AOF file from growing too large. --- redis.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/redis.conf b/redis.conf index 2a417579f..79d6804f5 100644 --- a/redis.conf +++ b/redis.conf @@ -2,6 +2,9 @@ bind 127.0.0.1 ::1 protected-mode yes port 6379 +auto-aof-rewrite-percentage 50 +auto-aof-rewrite-min-size 128mb + rename-command FLUSHDB "" rename-command FLUSHALL "" rename-command DEBUG "" From c4596544a341a5030fe02fe7b1df704f343a35ee Mon Sep 17 00:00:00 2001 From: Rohan Sureshkumar Date: Wed, 24 Jan 2024 19:18:46 +0530 Subject: [PATCH 042/123] Issue-3187: fix failing tests --- bookwyrm/views/feed.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/bookwyrm/views/feed.py b/bookwyrm/views/feed.py index 79e2f24d2..6e8f820e4 100644 --- a/bookwyrm/views/feed.py +++ b/bookwyrm/views/feed.py @@ -52,9 +52,18 @@ class Feed(View): paginated = Paginator(filtered_activities, PAGE_LENGTH) suggestions = suggested_users.get_suggestions(request.user) - cutoff = date(get_annual_summary_year(), 12, 31) - readthroughs = models.ReadThrough.objects.filter( - user=request.user, finish_date__lte=cutoff + + cutoff = ( + date(get_annual_summary_year(), 12, 31) + if get_annual_summary_year() + else None + ) + readthroughs = ( + models.ReadThrough.objects.filter( + user=request.user, finish_date__lte=cutoff + ) + if get_annual_summary_year() + else [] ) data = { From 2d4b11aaeedd9530ad4ddcfcea6f8aeb2b55dda9 Mon Sep 17 00:00:00 2001 From: Alexey Skobkin Date: Thu, 25 Jan 2024 01:50:10 +0300 Subject: [PATCH 043/123] Adding FictionBook format ("FB2", "FB3") to autocomplete options in "Get a copy" block. --- bookwyrm/static/js/autocomplete.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bookwyrm/static/js/autocomplete.js b/bookwyrm/static/js/autocomplete.js index a98cd9634..6836d356d 100644 --- a/bookwyrm/static/js/autocomplete.js +++ b/bookwyrm/static/js/autocomplete.js @@ -111,6 +111,10 @@ const tries = { }, }, f: { + b: { + 2: "FB2", + 3: "FB3", + }, l: { a: { c: "FLAC", From 82f9aa9da4ae68c2e042bbb981c89d8089cd39dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Wed, 24 Jan 2024 19:30:45 +0100 Subject: [PATCH 044/123] Set SESSION_COOKIE_AGE from environment, default to one month While we do wish for a longer maximum age (up to one year, see #3082), we only want to do that after termination of active sessions is implemented (see #2278). In the meantime, by reading and setting the variable from settings, we allow site admins to alter the default. --- bookwyrm/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 16241f9df..4af7afb14 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -30,6 +30,9 @@ RELEASE_API = env( PAGE_LENGTH = env.int("PAGE_LENGTH", 15) DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English") +# TODO: extend maximum age to 1 year once termination of active sessions +# is implemented (see bookwyrm-social#2278, bookwyrm-social#3082). +SESSION_COOKIE_AGE = env.int("SESSION_COOKIE_AGE", 3600 * 24 * 30) # 1 month JS_CACHE = "ac315a3b" From 80ad36e75b20213939852470506f12593353216c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Wed, 24 Jan 2024 19:54:55 +0100 Subject: [PATCH 045/123] Include SESSION_COOKIE_AGE in .env.example Suggested-by: Alexey Skobkin --- .env.example | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 20ce8240b..d0971660e 100644 --- a/.env.example +++ b/.env.example @@ -137,6 +137,10 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60 # and AWS_S3_CUSTOM_DOMAIN (if used) are added by default. # Value should be a comma-separated list of host names. CSP_ADDITIONAL_HOSTS= + # The last number here means "megabytes" # Increase if users are having trouble uploading BookWyrm export files. -DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 100) \ No newline at end of file +DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 100) + +# Time before being logged out (in seconds) +# SESSION_COOKIE_AGE=2592000 # current default: 30 days From 31babdfa510d88f89d08cbfb56de94cd8c0ac028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Fri, 26 Jan 2024 06:01:34 -0300 Subject: [PATCH 046/123] Always prefer shared inboxes when computing receipent lists This avoids duplicate submissions to remote instances when mentioning followers (i.e., `POST /user/foo/inbox` followed by `POST /inbox`, which results in two separate `add_status` tasks, and might generate duplicates in the target instance). --- bookwyrm/models/activitypub_mixin.py | 2 +- bookwyrm/tests/models/test_activitypub_mixin.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index d0a941f43..41772a162 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -153,7 +153,7 @@ class ActivitypubMixin: mentions = self.recipients if hasattr(self, "recipients") else [] # we always send activities to explicitly mentioned users' inboxes - recipients = [u.inbox for u in mentions or [] if not u.local] + recipients = [u.shared_inbox or u.inbox for u in mentions if not u.local] # unless it's a dm, all the followers should receive the activity if privacy != "direct": diff --git a/bookwyrm/tests/models/test_activitypub_mixin.py b/bookwyrm/tests/models/test_activitypub_mixin.py index cad970412..2f6fad76d 100644 --- a/bookwyrm/tests/models/test_activitypub_mixin.py +++ b/bookwyrm/tests/models/test_activitypub_mixin.py @@ -227,14 +227,18 @@ class ActivitypubMixins(TestCase): shared_inbox="http://example.com/inbox", outbox="https://example.com/users/nutria/outbox", ) - MockSelf = namedtuple("Self", ("privacy", "user")) - mock_self = MockSelf("public", self.local_user) + MockSelf = namedtuple("Self", ("privacy", "user", "recipients")) self.local_user.followers.add(self.remote_user) self.local_user.followers.add(another_remote_user) + mock_self = MockSelf("public", self.local_user, []) recipients = ActivitypubMixin.get_recipients(mock_self) - self.assertEqual(len(recipients), 1) - self.assertEqual(recipients[0], "http://example.com/inbox") + self.assertCountEqual(recipients, ["http://example.com/inbox"]) + + # should also work with recipient that is a follower + mock_self.recipients.append(another_remote_user) + recipients = ActivitypubMixin.get_recipients(mock_self) + self.assertCountEqual(recipients, ["http://example.com/inbox"]) def test_get_recipients_software(self, *_): """should differentiate between bookwyrm and other remote users""" From 8ac873419fe66de65cdddf6a25560ed24c4e4a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Fri, 26 Jan 2024 06:29:59 -0300 Subject: [PATCH 047/123] refactor: eagerly use a set in recipients, get_recipients --- bookwyrm/models/activitypub_mixin.py | 25 +++++++++++++------------ bookwyrm/models/status.py | 6 +++--- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index 41772a162..db737b8bc 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -152,8 +152,9 @@ class ActivitypubMixin: # find anyone who's tagged in a status, for example mentions = self.recipients if hasattr(self, "recipients") else [] - # we always send activities to explicitly mentioned users' inboxes - recipients = [u.shared_inbox or u.inbox for u in mentions if not u.local] + # we always send activities to explicitly mentioned users (using shared inboxes + # where available to avoid duplicate submissions to a given instance) + recipients = {u.shared_inbox or u.inbox for u in mentions if not u.local} # unless it's a dm, all the followers should receive the activity if privacy != "direct": @@ -173,18 +174,18 @@ class ActivitypubMixin: if user: queryset = queryset.filter(following=user) - # ideally, we will send to shared inboxes for efficiency - shared_inboxes = ( - queryset.filter(shared_inbox__isnull=False) - .values_list("shared_inbox", flat=True) - .distinct() + # as above, we prefer shared inboxes if available + recipients.update( + queryset.filter(shared_inbox__isnull=False).values_list( + "shared_inbox", flat=True + ) ) - # but not everyone has a shared inbox - inboxes = queryset.filter(shared_inbox__isnull=True).values_list( - "inbox", flat=True + recipients.update( + queryset.filter(shared_inbox__isnull=True).values_list( + "inbox", flat=True + ) ) - recipients += list(shared_inboxes) + list(inboxes) - return list(set(recipients)) + return list(recipients) def to_activity_dataclass(self): """convert from a model to an activity""" diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index cc44fe2bf..0d1d0d839 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -107,14 +107,14 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): @property def recipients(self): """tagged users who definitely need to get this status in broadcast""" - mentions = [u for u in self.mention_users.all() if not u.local] + mentions = {u for u in self.mention_users.all() if not u.local} if ( hasattr(self, "reply_parent") and self.reply_parent and not self.reply_parent.user.local ): - mentions.append(self.reply_parent.user) - return list(set(mentions)) + mentions.add(self.reply_parent.user) + return list(mentions) @classmethod def ignore_activity( From 940274b1c22e08fc870e92cc95e5ae6a95f5297e Mon Sep 17 00:00:00 2001 From: Braden Solt Date: Fri, 26 Jan 2024 15:47:55 -0700 Subject: [PATCH 048/123] classes that fix widths --- bookwyrm/templates/confirm_email/confirm_email.html | 4 ++-- bookwyrm/templates/landing/invite.html | 4 ++-- bookwyrm/templates/landing/login.html | 12 +++++++----- bookwyrm/templates/landing/password_reset.html | 4 ++-- bookwyrm/templates/landing/reactivate.html | 12 +++++++----- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/bookwyrm/templates/confirm_email/confirm_email.html b/bookwyrm/templates/confirm_email/confirm_email.html index abdd3a734..49a1ebd2d 100644 --- a/bookwyrm/templates/confirm_email/confirm_email.html +++ b/bookwyrm/templates/confirm_email/confirm_email.html @@ -6,8 +6,8 @@ {% block content %}

    {% trans "Confirm your email address" %}

    -
    -
    +
    +

    {% trans "A confirmation code has been sent to the email address you used to register your account." %}

    diff --git a/bookwyrm/templates/landing/invite.html b/bookwyrm/templates/landing/invite.html index d56cad38c..3e3ddab85 100644 --- a/bookwyrm/templates/landing/invite.html +++ b/bookwyrm/templates/landing/invite.html @@ -6,8 +6,8 @@ {% block content %}

    {% trans "Create an Account" %}

    -
    -
    +
    +
    {% if valid %}
    diff --git a/bookwyrm/templates/landing/login.html b/bookwyrm/templates/landing/login.html index 369a72bd2..8ea828b74 100644 --- a/bookwyrm/templates/landing/login.html +++ b/bookwyrm/templates/landing/login.html @@ -6,7 +6,7 @@ {% block content %}

    {% trans "Log in" %}

    -
    +
    {% if login_form.non_field_errors %}

    {{ login_form.non_field_errors }}

    {% endif %} @@ -20,13 +20,15 @@
    - +
    - +
    {% include 'snippets/form_errors.html' with errors_list=login_form.password.errors id="desc_password" %} @@ -58,10 +60,10 @@ {% include 'snippets/about.html' %}

    - {% trans "More about this site" %} + {% trans "More about this site" %}

    -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/bookwyrm/templates/landing/password_reset.html b/bookwyrm/templates/landing/password_reset.html index 786eaa0ab..2f41c5505 100644 --- a/bookwyrm/templates/landing/password_reset.html +++ b/bookwyrm/templates/landing/password_reset.html @@ -4,8 +4,8 @@ {% block title %}{% trans "Reset Password" %}{% endblock %} {% block content %} -
    -
    +
    +

    {% trans "Reset Password" %}

    diff --git a/bookwyrm/templates/landing/reactivate.html b/bookwyrm/templates/landing/reactivate.html index da9e0b050..adb41238f 100644 --- a/bookwyrm/templates/landing/reactivate.html +++ b/bookwyrm/templates/landing/reactivate.html @@ -6,7 +6,7 @@ {% block content %}

    {% trans "Reactivate Account" %}

    -
    +
    {% if login_form.non_field_errors %}

    {{ login_form.non_field_errors }}

    {% endif %} @@ -16,13 +16,15 @@
    - +
    - +
    {% include 'snippets/form_errors.html' with errors_list=login_form.password.errors id="desc_password" %} @@ -51,10 +53,10 @@ {% include 'snippets/about.html' %}

    - {% trans "More about this site" %} + {% trans "More about this site" %}

    -{% endblock %} +{% endblock %} \ No newline at end of file From 629acbaa193fa2b4a22593fd31a2e31f3bb8c3ff Mon Sep 17 00:00:00 2001 From: Braden Solt Date: Sat, 27 Jan 2024 10:58:57 -0700 Subject: [PATCH 049/123] add series number on posts in the feed --- bookwyrm/templates/snippets/book_titleby.html | 9 ++++++++- bookwyrm/templates/snippets/status/headers/comment.html | 4 ++++ .../templates/snippets/status/headers/quotation.html | 4 ++++ bookwyrm/templates/snippets/status/headers/read.html | 4 ++++ bookwyrm/templates/snippets/status/headers/reading.html | 4 ++++ bookwyrm/templates/snippets/status/headers/review.html | 4 ++++ .../snippets/status/headers/stopped_reading.html | 4 ++++ bookwyrm/templates/snippets/status/headers/to_read.html | 5 +++++ 8 files changed, 37 insertions(+), 1 deletion(-) diff --git a/bookwyrm/templates/snippets/book_titleby.html b/bookwyrm/templates/snippets/book_titleby.html index dc42bf273..35e0c4c45 100644 --- a/bookwyrm/templates/snippets/book_titleby.html +++ b/bookwyrm/templates/snippets/book_titleby.html @@ -9,12 +9,19 @@ {% if book.authors.exists %} {% blocktrans trimmed with path=book.local_path title=book|book_title %} -{{ title }} by +{{ title }} + +by {% endblocktrans %} {% include 'snippets/authors.html' with book=book limit=3 %} {% else %} {{ book|book_title }} {% endif %} +{% if book.series %} +({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) +{% endif %} + + {% endcache %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/comment.html b/bookwyrm/templates/snippets/status/headers/comment.html index e3e2ec40b..b78175bb0 100644 --- a/bookwyrm/templates/snippets/status/headers/comment.html +++ b/bookwyrm/templates/snippets/status/headers/comment.html @@ -16,5 +16,9 @@ commented on {{ book }} by {{ book }} {% endblocktrans %} +{% if book.series %} +({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) +{% endif %} + {% endif %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/quotation.html b/bookwyrm/templates/snippets/status/headers/quotation.html index 2cdd5a991..8a6776303 100644 --- a/bookwyrm/templates/snippets/status/headers/quotation.html +++ b/bookwyrm/templates/snippets/status/headers/quotation.html @@ -16,5 +16,9 @@ quoted {{ book }} by { quoted {{ book }} {% endblocktrans %} +{% if book.series %} +({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) +{% endif %} + {% endif %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/read.html b/bookwyrm/templates/snippets/status/headers/read.html index a59a3544e..aaebbd267 100644 --- a/bookwyrm/templates/snippets/status/headers/read.html +++ b/bookwyrm/templates/snippets/status/headers/read.html @@ -18,5 +18,9 @@ finished reading {{ book }} by {{ book }} {% endblocktrans %} +{% if book.series %} +({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) +{% endif %} + {% endif %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/reading.html b/bookwyrm/templates/snippets/status/headers/reading.html index 886158f29..f773775e0 100644 --- a/bookwyrm/templates/snippets/status/headers/reading.html +++ b/bookwyrm/templates/snippets/status/headers/reading.html @@ -12,6 +12,10 @@ started reading {{ book }} by {{ book }} by {{ book }} {% endblocktrans %} +{% if book.series %} +({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) +{% endif %} + {% endif %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/stopped_reading.html b/bookwyrm/templates/snippets/status/headers/stopped_reading.html index 3b6a314e1..d3891cb78 100644 --- a/bookwyrm/templates/snippets/status/headers/stopped_reading.html +++ b/bookwyrm/templates/snippets/status/headers/stopped_reading.html @@ -18,6 +18,10 @@ stopped reading {{ book }} by {{ book }} {% endblocktrans %} +{% if book.series %} +({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) +{% endif %} + {% endif %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/to_read.html b/bookwyrm/templates/snippets/status/headers/to_read.html index 2abdde17b..4d1b2ab1b 100644 --- a/bookwyrm/templates/snippets/status/headers/to_read.html +++ b/bookwyrm/templates/snippets/status/headers/to_read.html @@ -19,4 +19,9 @@ wants to read {{ book }} {% endblocktrans %} {% endif %} + +{% if book.series %} +({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) +{% endif %} + {% endspaceless %} From 6add81cf158405bdab718bba8378d3485cacb77d Mon Sep 17 00:00:00 2001 From: Braden Solt Date: Sat, 27 Jan 2024 11:02:42 -0700 Subject: [PATCH 050/123] move outside of authors "if" --- bookwyrm/templates/snippets/status/headers/comment.html | 3 ++- .../templates/snippets/status/headers/quotation.html | 3 ++- bookwyrm/templates/snippets/status/headers/read.html | 3 ++- bookwyrm/templates/snippets/status/headers/reading.html | 9 +++++---- bookwyrm/templates/snippets/status/headers/review.html | 3 ++- .../snippets/status/headers/stopped_reading.html | 3 ++- 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/bookwyrm/templates/snippets/status/headers/comment.html b/bookwyrm/templates/snippets/status/headers/comment.html index b78175bb0..b7750d3df 100644 --- a/bookwyrm/templates/snippets/status/headers/comment.html +++ b/bookwyrm/templates/snippets/status/headers/comment.html @@ -16,9 +16,10 @@ commented on {{ book }} by {{ book }} {% endblocktrans %} +{% endif %} + {% if book.series %} ({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) {% endif %} -{% endif %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/quotation.html b/bookwyrm/templates/snippets/status/headers/quotation.html index 8a6776303..3ddd20291 100644 --- a/bookwyrm/templates/snippets/status/headers/quotation.html +++ b/bookwyrm/templates/snippets/status/headers/quotation.html @@ -16,9 +16,10 @@ quoted {{ book }} by { quoted {{ book }} {% endblocktrans %} +{% endif %} + {% if book.series %} ({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) {% endif %} -{% endif %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/read.html b/bookwyrm/templates/snippets/status/headers/read.html index aaebbd267..8c323dac6 100644 --- a/bookwyrm/templates/snippets/status/headers/read.html +++ b/bookwyrm/templates/snippets/status/headers/read.html @@ -18,9 +18,10 @@ finished reading {{ book }} by {{ book }} {% endblocktrans %} +{% endif %} + {% if book.series %} ({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) {% endif %} -{% endif %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/reading.html b/bookwyrm/templates/snippets/status/headers/reading.html index f773775e0..73e96ebbc 100644 --- a/bookwyrm/templates/snippets/status/headers/reading.html +++ b/bookwyrm/templates/snippets/status/headers/reading.html @@ -12,10 +12,6 @@ started reading {{ book }} by {{ book }} {% endblocktrans %} {% endif %} + +{% if book.series %} +({{book.series}} {%if book.series_number %}#{{book.series_number}}) {% endif %} +{% endif %} + {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/review.html b/bookwyrm/templates/snippets/status/headers/review.html index 371d8b2cb..0bdcd29b1 100644 --- a/bookwyrm/templates/snippets/status/headers/review.html +++ b/bookwyrm/templates/snippets/status/headers/review.html @@ -16,9 +16,10 @@ reviewed {{ book }} by {{ book }} {% endblocktrans %} +{% endif %} + {% if book.series %} ({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) {% endif %} -{% endif %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/stopped_reading.html b/bookwyrm/templates/snippets/status/headers/stopped_reading.html index d3891cb78..a00be181c 100644 --- a/bookwyrm/templates/snippets/status/headers/stopped_reading.html +++ b/bookwyrm/templates/snippets/status/headers/stopped_reading.html @@ -18,10 +18,11 @@ stopped reading {{ book }} by {{ book }} {% endblocktrans %} +{% endif %} + {% if book.series %} ({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) {% endif %} -{% endif %} {% endspaceless %} From b05621005e14818fd4d529c06b38b4c433a7832d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Jan 2024 00:10:13 +0000 Subject: [PATCH 051/123] Bump aiohttp from 3.9.0 to 3.9.2 Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.9.0 to 3.9.2. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.9.0...v3.9.2) --- updated-dependencies: - dependency-name: aiohttp dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6509effc7..41b6bd16d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -aiohttp==3.9.0 +aiohttp==3.9.2 bleach==5.0.1 celery==5.2.7 colorthief==0.2.1 From 2ba7dff845e7211a22c390e32e590f84b16311fd Mon Sep 17 00:00:00 2001 From: Jacob Kerr Date: Wed, 3 Jan 2024 21:54:25 -0500 Subject: [PATCH 052/123] Fixed shelving date changing when changing editions --- bookwyrm/views/books/editions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bookwyrm/views/books/editions.py b/bookwyrm/views/books/editions.py index 54d1bd84c..5202531f5 100644 --- a/bookwyrm/views/books/editions.py +++ b/bookwyrm/views/books/editions.py @@ -93,6 +93,7 @@ def switch_edition(request): user=shelfbook.user, shelf=shelfbook.shelf, book=new_edition, + shelved_date=shelfbook.shelved_date, ) shelfbook.delete() From 9c5f6c527bd9ae1954a22d695f43f1c8f2c82907 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 3 Feb 2024 06:51:23 -0800 Subject: [PATCH 053/123] Fixes translation tags --- .../templates/preferences/export-user.html | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/bookwyrm/templates/preferences/export-user.html b/bookwyrm/templates/preferences/export-user.html index 58b27f3c1..955cff656 100644 --- a/bookwyrm/templates/preferences/export-user.html +++ b/bookwyrm/templates/preferences/export-user.html @@ -14,31 +14,29 @@

    {% trans "You can create an export file here. This will allow you to migrate your data to another BookWyrm account." %}

    - {% blocktrans trimmed %}
    -

    Your file will include:

    +

    {% trans "Your file will include:" %}

      -
    • User profile
      • -
    • Most user settings
      • -
    • Reading goals
      • -
    • Shelves
      • -
    • Reading history
      • -
    • Book reviews
      • -
    • Statuses
      • -
    • Your own lists and saved lists
      • -
    • Which users you follow and block
      • +
    • {% trans "User profile" %}
      • +
    • {% trans "Most user settings" %}
      • +
    • {% trans "Reading goals" %}
      • +
    • {% trans "Shelves" %}
      • +
    • {% trans "Reading history" %}
      • +
    • {% trans "Book reviews" %}
      • +
    • {% trans "Statuses" %}
      • +
    • {% trans "Your own lists and saved lists" %}
      • +
    • {% trans "Which users you follow and block" %}
    -

    Your file will not include:

    +

    {% trans "Your file will not include:" %}

      -
    • Direct messages
      • -
    • Replies to your statuses
      • -
    • Groups
      • -
    • Favorites
      • +
    • {% trans "Direct messages" %}
      • +
    • {% trans "Replies to your statuses" %}
      • +
    • {% trans "Groups" %}
      • +
    • {% trans "Favorites" %}
    - {% endblocktrans %}

    {% trans "In your new BookWyrm account can choose what to import: you will not have to import everything that is exported." %}

    @@ -50,11 +48,11 @@

    {% trans "New user exports are currently disabled." %} {% if perms.bookwyrm.edit_instance_settings %} - {% spaceless %} - {% blocktrans%} -
    User exports settings can be changed from the Imports page in the Admin dashboard. +
    + {% url 'settings-imports' as url %} + {% blocktrans trimmed %} + User exports settings can be changed from the Imports page in the Admin dashboard. {% endblocktrans %} - {% endspaceless %} {% endif%}

    {% elif next_available %} From 6d5752fb4ee72287ed15b84726872a4608842159 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 3 Feb 2024 07:40:23 -0800 Subject: [PATCH 054/123] Adds merge migration for page numbering fix --- bookwyrm/migrations/0193_merge_20240203_1539.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 bookwyrm/migrations/0193_merge_20240203_1539.py diff --git a/bookwyrm/migrations/0193_merge_20240203_1539.py b/bookwyrm/migrations/0193_merge_20240203_1539.py new file mode 100644 index 000000000..a88568ba1 --- /dev/null +++ b/bookwyrm/migrations/0193_merge_20240203_1539.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.23 on 2024-02-03 15:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0192_make_page_positions_text"), + ("bookwyrm", "0192_sitesettings_user_exports_enabled"), + ] + + operations = [] From a1ac9494b28ecd1a1674926d7aad8e82d2502dcf Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 3 Feb 2024 08:00:07 -0800 Subject: [PATCH 055/123] Allow admins to un-schedule tasks --- bookwyrm/templates/settings/schedules.html | 11 +++++++++++ bookwyrm/urls.py | 2 +- bookwyrm/views/admin/schedule.py | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/bookwyrm/templates/settings/schedules.html b/bookwyrm/templates/settings/schedules.html index fe096092d..20ced4b30 100644 --- a/bookwyrm/templates/settings/schedules.html +++ b/bookwyrm/templates/settings/schedules.html @@ -61,7 +61,18 @@ {{ task.interval.id }} + + {% if task.enabled %} + + {% endif %} {{ task.enabled|yesno }} + + {% if task.name != "celery.backend_cleanup" %} + + {% csrf_token %} + + + {% endif %} {% empty %} diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 64742347a..a40dcebea 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -360,7 +360,7 @@ urlpatterns = [ r"^settings/celery/ping/?$", views.celery_ping, name="settings-celery-ping" ), re_path( - r"^settings/schedules/?$", + r"^settings/schedules/(?P\d+)?$", views.ScheduledTasks.as_view(), name="settings-schedules", ), diff --git a/bookwyrm/views/admin/schedule.py b/bookwyrm/views/admin/schedule.py index ce5944ee5..c654dca9a 100644 --- a/bookwyrm/views/admin/schedule.py +++ b/bookwyrm/views/admin/schedule.py @@ -1,5 +1,6 @@ """ Scheduled celery tasks """ from django.contrib.auth.decorators import login_required, permission_required +from django.shortcuts import redirect from django.template.response import TemplateResponse from django.utils.decorators import method_decorator from django.views import View @@ -21,3 +22,10 @@ class ScheduledTasks(View): data["tasks"] = PeriodicTask.objects.all() data["schedules"] = IntervalSchedule.objects.all() return TemplateResponse(request, "settings/schedules.html", data) + + # pylint: disable=unused-argument + def post(self, request, task_id): + """un-schedule a task""" + task = PeriodicTask.objects.get(id=task_id) + task.delete() + return redirect("settings-schedules") From 4e2b8af1479bd35bf2b5c5973ff0ae2b4bb1a4fc Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 3 Feb 2024 08:02:51 -0800 Subject: [PATCH 056/123] Adds merge migration --- bookwyrm/migrations/0193_merge_20240203_1602.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 bookwyrm/migrations/0193_merge_20240203_1602.py diff --git a/bookwyrm/migrations/0193_merge_20240203_1602.py b/bookwyrm/migrations/0193_merge_20240203_1602.py new file mode 100644 index 000000000..e5f760539 --- /dev/null +++ b/bookwyrm/migrations/0193_merge_20240203_1602.py @@ -0,0 +1,13 @@ +# Generated by Django 3.2.23 on 2024-02-03 16:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0192_rename_version_sitesettings_available_version"), + ("bookwyrm", "0192_sitesettings_user_exports_enabled"), + ] + + operations = [] From 748c9349865f063689adcaa61e46a26fa3e3bcae Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 3 Feb 2024 08:20:12 -0800 Subject: [PATCH 057/123] Merge migrations upon merge migrations --- ...193_merge_20240203_1602.py => 0194_merge_20240203_1619.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename bookwyrm/migrations/{0193_merge_20240203_1602.py => 0194_merge_20240203_1619.py} (63%) diff --git a/bookwyrm/migrations/0193_merge_20240203_1602.py b/bookwyrm/migrations/0194_merge_20240203_1619.py similarity index 63% rename from bookwyrm/migrations/0193_merge_20240203_1602.py rename to bookwyrm/migrations/0194_merge_20240203_1619.py index e5f760539..a5c18e300 100644 --- a/bookwyrm/migrations/0193_merge_20240203_1602.py +++ b/bookwyrm/migrations/0194_merge_20240203_1619.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.23 on 2024-02-03 16:02 +# Generated by Django 3.2.23 on 2024-02-03 16:19 from django.db import migrations @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ ("bookwyrm", "0192_rename_version_sitesettings_available_version"), - ("bookwyrm", "0192_sitesettings_user_exports_enabled"), + ("bookwyrm", "0193_merge_20240203_1539"), ] operations = [] From 6c9ca0bf190552597890c2aaefca614c0058f914 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 3 Feb 2024 21:55:46 +0100 Subject: [PATCH 058/123] Add search for author --- bookwyrm/templates/layout.html | 2 +- bookwyrm/templates/search/author.html | 17 +++++++++++++++++ bookwyrm/templates/search/layout.html | 4 ++++ bookwyrm/views/search.py | 27 +++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 bookwyrm/templates/search/author.html diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index 6283e61c4..ced4e8006 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -36,7 +36,7 @@
    {% if request.user.is_authenticated %} - {% trans "Search for a book, user, or list" as search_placeholder %} + {% trans "Search for a book, author, user, or list" as search_placeholder %} {% else %} {% trans "Search for a book" as search_placeholder %} {% endif %} diff --git a/bookwyrm/templates/search/author.html b/bookwyrm/templates/search/author.html new file mode 100644 index 000000000..d42c3b54f --- /dev/null +++ b/bookwyrm/templates/search/author.html @@ -0,0 +1,17 @@ +{% extends 'search/layout.html' %} + +{% block panel %} + +{% if results %} + +{% endif %} + +{% endblock %} diff --git a/bookwyrm/templates/search/layout.html b/bookwyrm/templates/search/layout.html index 8cf47b371..725a4f43f 100644 --- a/bookwyrm/templates/search/layout.html +++ b/bookwyrm/templates/search/layout.html @@ -20,6 +20,7 @@
    + +
    {% endblock %} From dd1999eb8e75c17b525d301f46242b279ffb93df Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 20 Feb 2024 16:25:01 -0800 Subject: [PATCH 067/123] Adds view tests for shelf filters (#3162) * Adds test file * Adds success assertion * Updates tests * Updates shelf books creation * Updates assertion to use isbn for Edition model * Updates query * trigger workflow test * Updates validate_html * Updates comment and test * Fixes none test * Adds management command to clear all deleted user data * Adds success message --------- Co-authored-by: Mouse Reeve Co-authored-by: Mouse Reeve --- bookwyrm/tests/validate_html.py | 28 ++++++++++----- bookwyrm/tests/views/shelf/test_shelf.py | 45 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/bookwyrm/tests/validate_html.py b/bookwyrm/tests/validate_html.py index 85e5c6277..748b94d5f 100644 --- a/bookwyrm/tests/validate_html.py +++ b/bookwyrm/tests/validate_html.py @@ -13,16 +13,26 @@ def validate_html(html): "warn-proprietary-attributes": False, }, ) - # idk how else to filter out these unescape amp errs + # Tidy's parser is strict when validating unescaped/encoded ampersands found within + # the html document that are notpart of a character or entity reference + # (eg: `&` or `&`). Despite the fact the HTML5 spec no longer recommends + # escaping ampersands in URLs, Tidy will still complain if they are used as query + # param keys. Unfortunately, there is no way currently to configure tidy to ignore + # this so we must explictly redlist related strings that will appear in Tidy's + # errors output. + # + # See further discussion: https://github.com/htacg/tidy-html5/issues/1017 + excluded = [ + "&book", + "&type", + "&resolved", + "id and name attribute", + "illegal characters found in URI", + "escaping malformed URI reference", + "&filter", + ] errors = "\n".join( - e - for e in errors.split("\n") - if "&book" not in e - and "&type" not in e - and "&resolved" not in e - and "id and name attribute" not in e - and "illegal characters found in URI" not in e - and "escaping malformed URI reference" not in e + e for e in errors.split("\n") if not any(exclude in e for exclude in excluded) ) if errors: raise Exception(errors) diff --git a/bookwyrm/tests/views/shelf/test_shelf.py b/bookwyrm/tests/views/shelf/test_shelf.py index 492f214e3..b96d0a9ed 100644 --- a/bookwyrm/tests/views/shelf/test_shelf.py +++ b/bookwyrm/tests/views/shelf/test_shelf.py @@ -219,3 +219,48 @@ class ShelfViews(TestCase): view(request, request.user.username, shelf.identifier) self.assertEqual(shelf.name, "To Read") + + def test_filter_shelf_found(self, *_): + """display books that match a filter keyword""" + models.ShelfBook.objects.create( + book=self.book, + shelf=self.shelf, + user=self.local_user, + ) + shelf_book = models.ShelfBook.objects.create( + book=self.book, + shelf=self.local_user.shelf_set.first(), + user=self.local_user, + ) + view = views.Shelf.as_view() + request = self.factory.get("", {"filter": shelf_book.book.title}) + request.user = self.local_user + with patch("bookwyrm.views.shelf.shelf.is_api_request") as is_api: + is_api.return_value = False + result = view(request, self.local_user.username) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + self.assertEqual(len(result.context_data["books"].object_list), 1) + self.assertEqual( + result.context_data["books"].object_list[0].title, + shelf_book.book.title, + ) + + def test_filter_shelf_none(self, *_): + """display a message when no books match a filter keyword""" + models.ShelfBook.objects.create( + book=self.book, + shelf=self.shelf, + user=self.local_user, + ) + view = views.Shelf.as_view() + request = self.factory.get("", {"filter": "NOPE"}) + request.user = self.local_user + with patch("bookwyrm.views.shelf.shelf.is_api_request") as is_api: + is_api.return_value = False + result = view(request, self.local_user.username) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + self.assertEqual(len(result.context_data["books"].object_list), 0) From 8f537ef56a11fee392dd6e55688187ad95e6e91f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 20 Feb 2024 16:45:16 -0800 Subject: [PATCH 068/123] Adds missing migration for Korean locale --- .../0195_alter_user_preferred_language.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 bookwyrm/migrations/0195_alter_user_preferred_language.py diff --git a/bookwyrm/migrations/0195_alter_user_preferred_language.py b/bookwyrm/migrations/0195_alter_user_preferred_language.py new file mode 100644 index 000000000..1fbfa7304 --- /dev/null +++ b/bookwyrm/migrations/0195_alter_user_preferred_language.py @@ -0,0 +1,46 @@ +# Generated by Django 3.2.23 on 2024-02-21 00:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0194_merge_20240203_1619"), + ] + + operations = [ + migrations.AlterField( + model_name="user", + name="preferred_language", + field=models.CharField( + blank=True, + choices=[ + ("en-us", "English"), + ("ca-es", "Català (Catalan)"), + ("de-de", "Deutsch (German)"), + ("eo-uy", "Esperanto (Esperanto)"), + ("es-es", "Español (Spanish)"), + ("eu-es", "Euskara (Basque)"), + ("gl-es", "Galego (Galician)"), + ("it-it", "Italiano (Italian)"), + ("ko-kr", "한국어 (Korean)"), + ("fi-fi", "Suomi (Finnish)"), + ("fr-fr", "Français (French)"), + ("lt-lt", "Lietuvių (Lithuanian)"), + ("nl-nl", "Nederlands (Dutch)"), + ("no-no", "Norsk (Norwegian)"), + ("pl-pl", "Polski (Polish)"), + ("pt-br", "Português do Brasil (Brazilian Portuguese)"), + ("pt-pt", "Português Europeu (European Portuguese)"), + ("ro-ro", "Română (Romanian)"), + ("sv-se", "Svenska (Swedish)"), + ("uk-ua", "Українська (Ukrainian)"), + ("zh-hans", "简体中文 (Simplified Chinese)"), + ("zh-hant", "繁體中文 (Traditional Chinese)"), + ], + max_length=255, + null=True, + ), + ), + ] From db97d76a240919f31e48b8d97c1c51ca06aadbb4 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Fri, 1 Mar 2024 19:58:11 -0800 Subject: [PATCH 069/123] Add timeout to isbn.py An instance of requests.get in isbn.py lacks a timeout, and this commit adds one with a default of 15 as used other places in the code, where requests.get does already have a timeout. --- bookwyrm/isbn/isbn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/isbn/isbn.py b/bookwyrm/isbn/isbn.py index 56062ff7b..d14dc2619 100644 --- a/bookwyrm/isbn/isbn.py +++ b/bookwyrm/isbn/isbn.py @@ -26,7 +26,7 @@ class IsbnHyphenator: def update_range_message(self) -> None: """Download the range message xml file and save it locally""" - response = requests.get(self.__range_message_url) + response = requests.get(self.__range_message_url, timeout=15) with open(self.__range_file_path, "w", encoding="utf-8") as file: file.write(response.text) self.__element_tree = None From 1ae9870862626134f90b4f1fb86d06fdd870e6c4 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Fri, 1 Mar 2024 20:02:40 -0800 Subject: [PATCH 070/123] Add timeout to base_activity.py An instance of requests.get was missing a timeout; this commit adds a timeout of 15 as used in other places in this codebase which already have timeouts. --- bookwyrm/activitypub/base_activity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index fbbc18f73..9f1cfdbfb 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -423,6 +423,7 @@ def get_activitypub_data(url): "Date": now, "Signature": make_signature("get", sender, url, now), }, + timeout=15 ) except requests.RequestException: raise ConnectorException() From 50b811d9aa0e117f20851507b385b2020cfcd618 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Fri, 1 Mar 2024 20:11:14 -0800 Subject: [PATCH 071/123] Typo fix Add a comma --- bookwyrm/activitypub/base_activity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 9f1cfdbfb..890d4d24a 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -423,7 +423,7 @@ def get_activitypub_data(url): "Date": now, "Signature": make_signature("get", sender, url, now), }, - timeout=15 + timeout=15, ) except requests.RequestException: raise ConnectorException() From f8fd76cff05450249d59d76f02bc05e2138062dd Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 13:57:09 -0800 Subject: [PATCH 072/123] Remove duplicate types-requests==2.31.0.2 The types-requests==2.31.0.2 dependency was double-listed right next to each other; this commit removes one. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6dc737aab..931bc155b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,4 +54,3 @@ types-Pillow==10.0.0.3 types-psycopg2==2.9.21.11 types-python-dateutil==2.8.19.14 types-requests==2.31.0.2 -types-requests==2.31.0.2 From 3652ac81008fd2d0f8d640160de3178913237454 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 15:41:06 -0800 Subject: [PATCH 073/123] Alphabetize requirements.txt Alphabetize requirements.txt for developer convenience; this helps to find duplicates and unnecessarily-pinned subdependencies, as well as making the file easier to read and use. --- requirements.txt | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/requirements.txt b/requirements.txt index 931bc155b..f41210aea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,52 +1,52 @@ aiohttp==3.9.2 bleach==5.0.1 +boto3==1.26.57 +bw-file-resubmit==0.6.0rc2 celery==5.2.7 colorthief==0.2.1 Django==3.2.24 django-celery-beat==2.4.0 -bw-file-resubmit==0.6.0rc2 django-compressor==4.3.1 +django-csp==3.7 django-imagekit==4.1.0 django-model-utils==4.3.1 +django-redis==5.2.0 django-sass-processor==1.2.2 -django-csp==3.7 +django-storages==1.13.2 +django-storages[azure] environs==9.5.0 flower==1.2.0 libsass==0.22.0 Markdown==3.4.1 -Pillow==10.0.1 -psycopg2==2.9.5 -pycryptodome==3.19.1 -python-dateutil==2.8.2 -redis==4.5.4 -requests==2.31.0 -responses==0.22.0 -pytz>=2022.7 -boto3==1.26.57 -django-storages==1.13.2 -django-storages[azure] -django-redis==5.2.0 opentelemetry-api==1.16.0 opentelemetry-exporter-otlp-proto-grpc==1.16.0 opentelemetry-instrumentation-celery==0.37b0 opentelemetry-instrumentation-django==0.37b0 opentelemetry-instrumentation-psycopg2==0.37b0 opentelemetry-sdk==1.16.0 +Pillow==10.0.1 protobuf==3.20.* +psycopg2==2.9.5 +pycryptodome==3.19.1 pyotp==2.8.0 +python-dateutil==2.8.2 +pytz>=2022.7 qrcode==7.3.1 +redis==4.5.4 +requests==2.31.0 +responses==0.22.0 # Dev -pytest-django==4.1.0 +celery-types==0.18.0 +django-stubs[compatible-mypy]==4.2.4 +mypy==1.5.1 +pylint==2.14.0 pytest==6.1.2 pytest-cov==2.10.1 +pytest-django==4.1.0 pytest-env==0.6.2 pytest-xdist==2.3.0 pytidylib==0.3.2 -pylint==2.14.0 -mypy==1.5.1 -celery-types==0.18.0 -django-stubs[compatible-mypy]==4.2.4 types-bleach==6.0.0.4 types-dataclasses==0.6.6 types-Markdown==3.4.2.10 From 570017d3b08a700a7cd12c656a5eeeff059b466b Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 15:57:06 -0800 Subject: [PATCH 074/123] Upgrade Python Version from 3.9 to 3.11 --- .github/workflows/django-tests.yml | 2 +- .github/workflows/mypy.yml | 4 ++-- .github/workflows/pylint.yml | 4 ++-- Dockerfile | 2 +- dev-tools/Dockerfile | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index 78b6e142e..de71d9bcf 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.11 - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 1a641edd2..6df987aa4 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,10 +13,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.11 - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 3811c97d3..ab8633b48 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -13,10 +13,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.11 - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/Dockerfile b/Dockerfile index b3cd26e88..82b0c92c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.11 ENV PYTHONUNBUFFERED 1 diff --git a/dev-tools/Dockerfile b/dev-tools/Dockerfile index 6c132944f..563467f09 100644 --- a/dev-tools/Dockerfile +++ b/dev-tools/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-bookworm +FROM python:3.11-bookworm WORKDIR /app/dev-tools ENV PATH="/app/dev-tools/node_modules/.bin:$PATH" From 39da471f795fd99e40f68e0f48c831f56864208a Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 15:59:17 -0800 Subject: [PATCH 075/123] Disable Pylint Failure for imghdr deprecation for now --- bookwyrm/connectors/abstract_connector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 8b6dcb885..b61bc2b04 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -3,7 +3,9 @@ from __future__ import annotations from abc import ABC, abstractmethod from typing import Optional, TypedDict, Any, Callable, Union, Iterator from urllib.parse import quote_plus -import imghdr + +# pylint: disable-next=deprecated-module +import imghdr # Deprecated in 3.11 for removal in 3.13; no good alternative yet import logging import re import asyncio From 4312e9bba02df710ca0f11f40fffe00ef7af95b8 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 16:03:19 -0800 Subject: [PATCH 076/123] Upgrade Celery to 5.3.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f41210aea..0cfa9db9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aiohttp==3.9.2 bleach==5.0.1 boto3==1.26.57 bw-file-resubmit==0.6.0rc2 -celery==5.2.7 +celery==5.3.1 colorthief==0.2.1 Django==3.2.24 django-celery-beat==2.4.0 From c944824ac7abb6233b43a0f4cc573343635d287e Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 16:04:06 -0800 Subject: [PATCH 077/123] Upgrade django-celery-beat to 2.5.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0cfa9db9c..19f13307f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ bw-file-resubmit==0.6.0rc2 celery==5.3.1 colorthief==0.2.1 Django==3.2.24 -django-celery-beat==2.4.0 +django-celery-beat==2.5.0 django-compressor==4.3.1 django-csp==3.7 django-imagekit==4.1.0 From fee3fdd5a804a7cb3b234763649034b01b4f5d87 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 16:04:37 -0800 Subject: [PATCH 078/123] Upgrade django-compressor to 4.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 19f13307f..5882607ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ celery==5.3.1 colorthief==0.2.1 Django==3.2.24 django-celery-beat==2.5.0 -django-compressor==4.3.1 +django-compressor==4.4 django-csp==3.7 django-imagekit==4.1.0 django-model-utils==4.3.1 From c1520da56d0959a5b56f272c2803120aae696425 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 16:05:11 -0800 Subject: [PATCH 079/123] Upgrade flower to 2.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5882607ee..574c7811c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ django-sass-processor==1.2.2 django-storages==1.13.2 django-storages[azure] environs==9.5.0 -flower==1.2.0 +flower==2.0.0 libsass==0.22.0 Markdown==3.4.1 opentelemetry-api==1.16.0 From da2636fa294eec47502666e21dc41e4f71c66b4c Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 16:07:50 -0800 Subject: [PATCH 080/123] Add grpcio pin @ 1.57.0 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 574c7811c..076456b0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ django-storages==1.13.2 django-storages[azure] environs==9.5.0 flower==2.0.0 +grpcio=1.57.0 # Not a direct dependency, pinned to get a security fix libsass==0.22.0 Markdown==3.4.1 opentelemetry-api==1.16.0 From 0f5a3e9163a3f52843b222668dd7a273014db500 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 16:08:41 -0800 Subject: [PATCH 081/123] Pin Tornado at 6.3.3 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 076456b0d..05c606a83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ qrcode==7.3.1 redis==4.5.4 requests==2.31.0 responses==0.22.0 +tornado==6.3.3 # Not a direct dependency, pinned to get a security fix # Dev celery-types==0.18.0 From 498dc35d995793a73b3ae358a3d1f2d969240684 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 16:09:06 -0800 Subject: [PATCH 082/123] Upgrade Pylint to 2.15.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 05c606a83..a2f95c6bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,7 +42,7 @@ tornado==6.3.3 # Not a direct dependency, pinned to get a security fix celery-types==0.18.0 django-stubs[compatible-mypy]==4.2.4 mypy==1.5.1 -pylint==2.14.0 +pylint==2.15.0 pytest==6.1.2 pytest-cov==2.10.1 pytest-django==4.1.0 From 22c4155c7c4b850c538ce26a5590d02acc4d706b Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 16:09:34 -0800 Subject: [PATCH 083/123] Upgrade pytest to 6.2.5 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a2f95c6bc..ae73b1d61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ celery-types==0.18.0 django-stubs[compatible-mypy]==4.2.4 mypy==1.5.1 pylint==2.15.0 -pytest==6.1.2 +pytest==6.2.5 pytest-cov==2.10.1 pytest-django==4.1.0 pytest-env==0.6.2 From be140d5e5a5818b73c40c7259a54a510681789db Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 17:20:48 -0800 Subject: [PATCH 084/123] Pin setuptools at 65.5.1 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index ae73b1d61..bdf84f7e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ qrcode==7.3.1 redis==4.5.4 requests==2.31.0 responses==0.22.0 +setuptools>=65.5.1 # Not a direct dependency, pinned to get a security fix tornado==6.3.3 # Not a direct dependency, pinned to get a security fix # Dev From eadb0e640f84f9a69c39638ce425e78fadffb594 Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 17:29:42 -0800 Subject: [PATCH 085/123] Fix typo in operator --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bdf84f7e7..c769916a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ django-storages==1.13.2 django-storages[azure] environs==9.5.0 flower==2.0.0 -grpcio=1.57.0 # Not a direct dependency, pinned to get a security fix +grpcio==1.57.0 # Not a direct dependency, pinned to get a security fix libsass==0.22.0 Markdown==3.4.1 opentelemetry-api==1.16.0 From 9fa09d5ebeb96a379f1ba6765aaf22324b8f631d Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 17:30:37 -0800 Subject: [PATCH 086/123] Add extra space required by linter --- bookwyrm/connectors/abstract_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index b61bc2b04..fbf0fb31b 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -5,7 +5,7 @@ from typing import Optional, TypedDict, Any, Callable, Union, Iterator from urllib.parse import quote_plus # pylint: disable-next=deprecated-module -import imghdr # Deprecated in 3.11 for removal in 3.13; no good alternative yet +import imghdr # Deprecated in 3.11 for removal in 3.13; no good alternative yet import logging import re import asyncio From 91fe4ad535fbe4717042c3189d38d7b4c1336f4a Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 17:31:16 -0800 Subject: [PATCH 087/123] Fix spacing for linter --- bookwyrm/connectors/abstract_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index fbf0fb31b..aa8edbeae 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -5,7 +5,7 @@ from typing import Optional, TypedDict, Any, Callable, Union, Iterator from urllib.parse import quote_plus # pylint: disable-next=deprecated-module -import imghdr # Deprecated in 3.11 for removal in 3.13; no good alternative yet +import imghdr # Deprecated in 3.11 for removal in 3.13; no good alternative yet import logging import re import asyncio From d138395c75a6790f6ad81a727ad8a3f2c8b1169d Mon Sep 17 00:00:00 2001 From: Margaret Fero Date: Sat, 2 Mar 2024 17:43:49 -0800 Subject: [PATCH 088/123] Add linter exclusion for TBookWyrmModel --- bookwyrm/activitypub/base_activity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 890d4d24a..efc9d8da2 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -20,6 +20,7 @@ from bookwyrm.tasks import app, MISC logger = logging.getLogger(__name__) +# pylint: disable=invalid-name TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel) From 09c3d9c0dcdb821de612ac9f47819599bf349dcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sun, 3 Mar 2024 18:42:27 -0300 Subject: [PATCH 089/123] json_export: also detect absent "icon" key --- bookwyrm/models/bookwyrm_export_job.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index 1f6085e0c..2f32cbd29 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -80,10 +80,7 @@ def json_export( exported_user = user.to_activity() # I don't love this but it prevents a JSON encoding error # when there is no user image - if isinstance( - exported_user["icon"], - dataclasses._MISSING_TYPE, # pylint: disable=protected-access - ): + if exported_user.get("icon") in (None, dataclasses.MISSING): exported_user["icon"] = {} else: # change the URL to be relative to the JSON file From 8d018b872fe33ccdd729a7fb953f65885a45cd47 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 9 Mar 2024 15:39:52 +0100 Subject: [PATCH 090/123] FileLinkForm: fix duplicate check --- bookwyrm/forms/links.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bookwyrm/forms/links.py b/bookwyrm/forms/links.py index d2fd5f116..345c5c1d4 100644 --- a/bookwyrm/forms/links.py +++ b/bookwyrm/forms/links.py @@ -1,4 +1,5 @@ """ using django model forms """ + from urllib.parse import urlparse from django.utils.translation import gettext_lazy as _ @@ -37,10 +38,9 @@ class FileLinkForm(CustomForm): ), ) if ( - not self.instance - and models.FileLink.objects.filter( - url=url, book=book, filetype=filetype - ).exists() + models.FileLink.objects.filter(url=url, book=book, filetype=filetype) + .exclude(pk=self.instance) + .exists() ): # pylint: disable=line-too-long self.add_error( From 3ba528ecddc08bf5f18c8596da2b4fd9f0758291 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 11 Mar 2024 12:46:55 +0100 Subject: [PATCH 091/123] pytest.ini: define ALLOWED_HOSTS This fixes running `./bw-dev pytest` locally when having a different value defined for `ALLOWED_HOSTS` in `.env`. --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index b50efd602..18c955032 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,6 +11,7 @@ env = DEBUG = false USE_HTTPS = true DOMAIN = your.domain.here + ALLOWED_HOSTS = your.domain.here BOOKWYRM_DATABASE_BACKEND = postgres MEDIA_ROOT = images/ CELERY_BROKER = "" From 12b469a0d6a2c554715f53bcf2a333d16dbed8f2 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 13 Mar 2024 12:30:52 +0100 Subject: [PATCH 092/123] CI: use actions/checkout@v4 --- .github/workflows/black.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/curlylint.yaml | 2 +- .github/workflows/django-tests.yml | 2 +- .github/workflows/lint-frontend.yaml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/prettier.yaml | 2 +- .github/workflows/pylint.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 4e7be4af3..7ac208c94 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -10,7 +10,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 - uses: psf/black@22.12.0 with: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 68bb05d7e..51316ef62 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/curlylint.yaml b/.github/workflows/curlylint.yaml index 8d5c6b4f7..10ad04ce1 100644 --- a/.github/workflows/curlylint.yaml +++ b/.github/workflows/curlylint.yaml @@ -10,7 +10,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install curlylint run: pip install curlylint diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index de71d9bcf..9a2c615a4 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -23,7 +23,7 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/lint-frontend.yaml b/.github/workflows/lint-frontend.yaml index 0d0559e40..21f11ebf3 100644 --- a/.github/workflows/lint-frontend.yaml +++ b/.github/workflows/lint-frontend.yaml @@ -19,7 +19,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install modules run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 6df987aa4..a198efc21 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.11 uses: actions/setup-python@v4 with: diff --git a/.github/workflows/prettier.yaml b/.github/workflows/prettier.yaml index 501516ae1..9c05c7476 100644 --- a/.github/workflows/prettier.yaml +++ b/.github/workflows/prettier.yaml @@ -14,7 +14,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install modules run: npm install prettier@2.5.1 diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index ab8633b48..85f275020 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.11 uses: actions/setup-python@v4 with: From 6af0a0883827b65e5776c2d1e2f7264600ae7d9f Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 13 Mar 2024 12:35:23 +0100 Subject: [PATCH 093/123] CI: use actions/setup-python@v5 and cache pip --- .github/workflows/black.yml | 2 +- .github/workflows/django-tests.yml | 3 ++- .github/workflows/mypy.yml | 3 ++- .github/workflows/pylint.yml | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 7ac208c94..0633dedb7 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 - uses: psf/black@22.12.0 with: version: 22.12.0 diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index 9a2c615a4..7d9cb3cba 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -25,9 +25,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 + cache: pip - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index a198efc21..d1e3f9fc9 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -14,9 +14,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 + cache: pip - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 85f275020..915e3154c 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -14,9 +14,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 + cache: pip - name: Install Dependencies run: | python -m pip install --upgrade pip From 74fdd9a85a8741ea5ac71432ebaffd6b800f8d23 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 13 Mar 2024 10:31:09 +0100 Subject: [PATCH 094/123] CI: simplify pytest setup --- .github/workflows/django-tests.yml | 52 +++++++++--------------------- 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index 7d9cb3cba..da237592f 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -7,12 +7,20 @@ on: jobs: build: - - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + env: # overrides for .env.example + POSTGRES_HOST: 127.0.0.1 + PGPORT: 5432 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: hunter2 + POSTGRES_DB: github_actions + SECRET_KEY: beepbeep + EMAIL_HOST_USER: "" + EMAIL_HOST_PASSWORD: "" services: postgres: image: postgres:13 - env: + env: # does not inherit from jobs.build.env POSTGRES_USER: postgres POSTGRES_PASSWORD: hunter2 options: >- @@ -33,39 +41,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + - name: Set up .env + run: cp .env.example .env - name: Check migrations up-to-date - run: | - python ./manage.py makemigrations --check - env: - SECRET_KEY: beepbeep - DOMAIN: your.domain.here - EMAIL_HOST: "" - EMAIL_HOST_USER: "" - EMAIL_HOST_PASSWORD: "" + run: python ./manage.py makemigrations --check - name: Run Tests - env: - SECRET_KEY: beepbeep - DEBUG: false - USE_HTTPS: true - DOMAIN: your.domain.here - BOOKWYRM_DATABASE_BACKEND: postgres - MEDIA_ROOT: images/ - POSTGRES_PASSWORD: hunter2 - POSTGRES_USER: postgres - POSTGRES_DB: github_actions - POSTGRES_HOST: 127.0.0.1 - CELERY_BROKER: "" - REDIS_BROKER_PORT: 6379 - REDIS_BROKER_PASSWORD: beep - USE_DUMMY_CACHE: true - FLOWER_PORT: 8888 - EMAIL_HOST: "smtp.mailgun.org" - EMAIL_PORT: 587 - EMAIL_HOST_USER: "" - EMAIL_HOST_PASSWORD: "" - EMAIL_USE_TLS: true - ENABLE_PREVIEW_IMAGES: false - ENABLE_THUMBNAIL_GENERATION: true - HTTP_X_FORWARDED_PROTO: false - run: | - pytest -n 3 + run: pytest -n 3 From 383e6533e11da5c88ceebb9cfad89d6980f7d430 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 13 Mar 2024 11:56:16 +0100 Subject: [PATCH 095/123] CI: use pytest-github-actions-annotate-failures --- .github/workflows/django-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index da237592f..ce9b2b7b1 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -41,6 +41,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install pytest-github-actions-annotate-failures - name: Set up .env run: cp .env.example .env - name: Check migrations up-to-date From 4e20e430379a820c1e4f88590bf6afe08cba8249 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 13 Mar 2024 12:48:02 +0100 Subject: [PATCH 096/123] CI: merge all Python actions into one file --- .github/workflows/black.yml | 17 ----- .github/workflows/django-tests.yml | 50 --------------- .github/workflows/mypy.yml | 51 --------------- .github/workflows/pylint.yml | 28 --------- .github/workflows/python.yml | 99 ++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 146 deletions(-) delete mode 100644 .github/workflows/black.yml delete mode 100644 .github/workflows/django-tests.yml delete mode 100644 .github/workflows/mypy.yml delete mode 100644 .github/workflows/pylint.yml create mode 100644 .github/workflows/python.yml diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml deleted file mode 100644 index 0633dedb7..000000000 --- a/.github/workflows/black.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Python Formatting (run ./bw-dev black to fix) - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - - uses: psf/black@22.12.0 - with: - version: 22.12.0 diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml deleted file mode 100644 index ce9b2b7b1..000000000 --- a/.github/workflows/django-tests.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Run Python Tests -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - runs-on: ubuntu-latest - env: # overrides for .env.example - POSTGRES_HOST: 127.0.0.1 - PGPORT: 5432 - POSTGRES_USER: postgres - POSTGRES_PASSWORD: hunter2 - POSTGRES_DB: github_actions - SECRET_KEY: beepbeep - EMAIL_HOST_USER: "" - EMAIL_HOST_PASSWORD: "" - services: - postgres: - image: postgres:13 - env: # does not inherit from jobs.build.env - POSTGRES_USER: postgres - POSTGRES_PASSWORD: hunter2 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - cache: pip - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest-github-actions-annotate-failures - - name: Set up .env - run: cp .env.example .env - - name: Check migrations up-to-date - run: python ./manage.py makemigrations --check - - name: Run Tests - run: pytest -n 3 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml deleted file mode 100644 index d1e3f9fc9..000000000 --- a/.github/workflows/mypy.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Mypy - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: 3.11 - cache: pip - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Analysing the code with mypy - env: - SECRET_KEY: beepbeep - DEBUG: false - USE_HTTPS: true - DOMAIN: your.domain.here - BOOKWYRM_DATABASE_BACKEND: postgres - MEDIA_ROOT: images/ - POSTGRES_PASSWORD: hunter2 - POSTGRES_USER: postgres - POSTGRES_DB: github_actions - POSTGRES_HOST: 127.0.0.1 - CELERY_BROKER: "" - REDIS_BROKER_PORT: 6379 - REDIS_BROKER_PASSWORD: beep - USE_DUMMY_CACHE: true - FLOWER_PORT: 8888 - EMAIL_HOST: "smtp.mailgun.org" - EMAIL_PORT: 587 - EMAIL_HOST_USER: "" - EMAIL_HOST_PASSWORD: "" - EMAIL_USE_TLS: true - ENABLE_PREVIEW_IMAGES: false - ENABLE_THUMBNAIL_GENERATION: true - HTTP_X_FORWARDED_PROTO: false - run: | - mypy bookwyrm celerywyrm diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 915e3154c..000000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Pylint - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: 3.11 - cache: pip - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Analysing the code with pylint - run: | - pylint bookwyrm/ - diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 000000000..dcbe05aee --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,99 @@ +name: Python +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +# overrides for .env.example +env: + POSTGRES_HOST: 127.0.0.1 + PGPORT: 5432 + POSTGRES_USER: postgres + POSTGRES_PASSWORD: hunter2 + POSTGRES_DB: github_actions + SECRET_KEY: beepbeep + EMAIL_HOST_USER: "" + EMAIL_HOST_PASSWORD: "" + +jobs: + pytest: + name: Tests (pytest) + runs-on: ubuntu-latest + services: + postgres: + image: postgres:13 + env: # does not inherit from jobs.build.env + POSTGRES_USER: postgres + POSTGRES_PASSWORD: hunter2 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: pip + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest-github-actions-annotate-failures + - name: Set up .env + run: cp .env.example .env + - name: Check migrations up-to-date + run: python ./manage.py makemigrations --check + - name: Run Tests + run: pytest -n 3 + + pylint: + name: Linting (pylint) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: pip + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Analyse code with pylint + run: pylint bookwyrm/ + + mypy: + name: Typing (mypy) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: pip + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Set up .env + run: cp .env.example .env + - name: Analyse code with mypy + run: mypy bookwyrm celerywyrm + + black: + name: Formatting (black; run ./bw-dev black to fix) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: psf/black@22.12.0 + with: + version: 22.12.0 From beb49af51492e310c433eb4ed23bb947fc52787e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sun, 17 Mar 2024 21:46:34 -0300 Subject: [PATCH 097/123] Upgade django-pgtrigger to 4.11 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 05fd9d2b4..90ff18edb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ django-celery-beat==2.4.0 django-compressor==4.3.1 django-imagekit==4.1.0 django-model-utils==4.3.1 -django-pgtrigger==4.10.0 +django-pgtrigger==4.11.0 django-sass-processor==1.2.2 django-csp==3.7 environs==9.5.0 From 2cf7ed477df2dddefa899a0feaca9e917f2b4738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sun, 17 Mar 2024 22:37:28 -0300 Subject: [PATCH 098/123] Consolidate test_posgres.py into test_book_search.py These are tests I missed when first writing trigger tests in test_book_search.py. --- bookwyrm/tests/test_book_search.py | 60 ++++++++++++++++++++++- bookwyrm/tests/test_postgres.py | 77 ------------------------------ 2 files changed, 58 insertions(+), 79 deletions(-) delete mode 100644 bookwyrm/tests/test_postgres.py diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index 439d2edbf..0721cb142 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -184,11 +184,67 @@ class SearchVectorTest(TestCase): book = self._create_book("Hear We Come", "John") self.assertEqual(book.search_vector, "'come':3A 'hear':1A 'john':4C") + book = self._create_book("there there", "the") + self.assertEqual(book.search_vector, "'the':3C 'there':1A,2A") + def test_search_vector_no_author(self): """book with no authors gets processed normally""" book = self._create_book("Book", None, series="Bunch") self.assertEqual(book.search_vector, "'book':1A 'bunch':2") + book = self._create_book("there there", None) + self.assertEqual(book.search_vector, "'there':1A,2A") + + # n.b.: the following originally from test_posgres.py + + def test_search_vector_on_update(self): + """make sure that search_vector is being set correctly on edit""" + book = self._create_book("The Long Goodbye", None) + self.assertEqual(book.search_vector, "'goodby':3A 'long':2A") + + book.title = "The Even Longer Goodbye" + book.save(broadcast=False) + book.refresh_from_db() + self.assertEqual(book.search_vector, "'even':2A 'goodby':4A 'longer':3A") + + def test_search_vector_on_author_update(self): + """update search when an author name changes""" + book = self._create_book("The Long Goodbye", "The Rays") + self.assertEqual(book.search_vector, "'goodby':3A 'long':2A 'rays':5C 'the':4C") + + author = models.Author.objects.get(name="The Rays") + author.name = "Jeremy" + author.save(broadcast=False) + book.refresh_from_db() + self.assertEqual(book.search_vector, "'goodby':3A 'jeremy':4C 'long':2A") + + def test_search_vector_on_author_delete(self): + """update search when an author is deleted""" + book = self._create_book("The Long Goodbye", "The Rays") + self.assertEqual(book.search_vector, "'goodby':3A 'long':2A 'rays':5C 'the':4C") + + author = models.Author.objects.get(name="The Rays") + book.authors.remove(author) + book.refresh_from_db() + self.assertEqual(book.search_vector, "'goodby':3A 'long':2A") + + def test_search_vector_fields(self): + """language field irrelevant for search_vector""" + author = models.Author.objects.create(name="The Rays") + book = models.Edition.objects.create( + title="The Long Goodbye", + subtitle="wow cool", + series="series name", + languages=["irrelevant"], + ) + book.authors.add(author) + book.refresh_from_db() + self.assertEqual( + book.search_vector, + # pylint: disable-next=line-too-long + "'cool':5B 'goodby':3A 'long':2A 'name':9 'rays':7C 'seri':8 'the':6C 'wow':4B", + ) + @staticmethod def _create_book( title, author_name, /, *, subtitle="", series="", author_alias=None @@ -212,8 +268,8 @@ class SearchVectorTest(TestCase): return edition -class SearchVectorTriggers(TestCase): - """look for books as they change""" +class SearchVectorUpdates(TestCase): + """look for books as they change""" # functional tests of the above def setUp(self): """we need basic test data and mocks""" diff --git a/bookwyrm/tests/test_postgres.py b/bookwyrm/tests/test_postgres.py deleted file mode 100644 index 8fc3c9d59..000000000 --- a/bookwyrm/tests/test_postgres.py +++ /dev/null @@ -1,77 +0,0 @@ -""" django configuration of postgres """ -from unittest.mock import patch -from django.test import TestCase - -from bookwyrm import models - - -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") -class PostgresTriggers(TestCase): - """special migrations, fancy stuff ya know""" - - def test_search_vector_on_create(self, _): - """make sure that search_vector is being set correctly on create""" - book = models.Edition.objects.create(title="The Long Goodbye") - book.refresh_from_db() - self.assertEqual(book.search_vector, "'goodby':3A 'long':2A") - - def test_search_vector_on_update(self, _): - """make sure that search_vector is being set correctly on edit""" - book = models.Edition.objects.create(title="The Long Goodbye") - book.title = "The Even Longer Goodbye" - book.save(broadcast=False) - book.refresh_from_db() - self.assertEqual(book.search_vector, "'even':2A 'goodby':4A 'longer':3A") - - def test_search_vector_fields(self, _): - """use multiple fields to create search vector""" - author = models.Author.objects.create(name="The Rays") - book = models.Edition.objects.create( - title="The Long Goodbye", - subtitle="wow cool", - series="series name", - languages=["irrelevant"], - ) - book.authors.add(author) - book.refresh_from_db() - # pylint: disable=line-too-long - self.assertEqual( - book.search_vector, - "'cool':5B 'goodby':3A 'long':2A 'name':9 'rays':7C 'seri':8 'the':6C 'wow':4B", - ) - - def test_search_vector_on_author_update(self, _): - """update search when an author name changes""" - author = models.Author.objects.create(name="The Rays") - book = models.Edition.objects.create( - title="The Long Goodbye", - ) - book.authors.add(author) - author.name = "Jeremy" - author.save(broadcast=False) - book.refresh_from_db() - - self.assertEqual(book.search_vector, "'goodby':3A 'jeremy':4C 'long':2A") - - def test_search_vector_on_author_delete(self, _): - """update search when an author name changes""" - author = models.Author.objects.create(name="Jeremy") - book = models.Edition.objects.create( - title="The Long Goodbye", - ) - - book.authors.add(author) - book.refresh_from_db() - self.assertEqual(book.search_vector, "'goodby':3A 'jeremy':4C 'long':2A") - - book.authors.remove(author) - book.refresh_from_db() - self.assertEqual(book.search_vector, "'goodby':3A 'long':2A") - - def test_search_vector_stop_word_fallback(self, _): - """use a fallback when removing stop words leads to an empty vector""" - book = models.Edition.objects.create( - title="there there", - ) - book.refresh_from_db() - self.assertEqual(book.search_vector, "'there':1A,2A") From 3be227fc86c104ccc115bc373b92bf7ef4a8d371 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 22:00:54 +0000 Subject: [PATCH 099/123] Bump pillow from 10.0.1 to 10.2.0 Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.0.1 to 10.2.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/10.0.1...10.2.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c769916a2..d59a62c98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ opentelemetry-instrumentation-celery==0.37b0 opentelemetry-instrumentation-django==0.37b0 opentelemetry-instrumentation-psycopg2==0.37b0 opentelemetry-sdk==1.16.0 -Pillow==10.0.1 +Pillow==10.2.0 protobuf==3.20.* psycopg2==2.9.5 pycryptodome==3.19.1 From ccf2b16d735679cb3349e449612b61d1b9141676 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 18 Mar 2024 19:52:40 +0100 Subject: [PATCH 100/123] requirements.txt: make typing-Pillow match Pillow --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d59a62c98..5e8611076 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,7 +53,7 @@ pytidylib==0.3.2 types-bleach==6.0.0.4 types-dataclasses==0.6.6 types-Markdown==3.4.2.10 -types-Pillow==10.0.0.3 +types-Pillow==10.2.0.20240311 types-psycopg2==2.9.21.11 types-python-dateutil==2.8.19.14 types-requests==2.31.0.2 From 748418590fa4bbfd20615a2114c00e322ce8b581 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 18 Mar 2024 20:22:14 +0100 Subject: [PATCH 101/123] docker-compose.yml: mount static_volume for flower Because flower also uses BookwyrmConfig, it wants to download fonts, and will download them to an incorrect location if the static_volume is not mounted. --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4d4037681..2cb9007da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,7 @@ services: env_file: .env volumes: - .:/app + - static_volume:/app/static networks: - main depends_on: From 3367b20965d44b5f2724250491e0b54c84fc3769 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 18 Mar 2024 20:23:26 +0100 Subject: [PATCH 102/123] Font download: destination dir is allowed to exist Without this argument, an existing directory (but not the file) causes an error. --- bookwyrm/apps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookwyrm/apps.py b/bookwyrm/apps.py index b0c3e3fa4..5a9f45db5 100644 --- a/bookwyrm/apps.py +++ b/bookwyrm/apps.py @@ -1,4 +1,5 @@ """Do further startup configuration and initialization""" + import os import urllib import logging @@ -14,7 +15,7 @@ def download_file(url, destination): """Downloads a file to the given path""" try: # Ensure our destination directory exists - os.makedirs(os.path.dirname(destination)) + os.makedirs(os.path.dirname(destination), exist_ok=True) with urllib.request.urlopen(url) as stream: with open(destination, "b+w") as outfile: outfile.write(stream.read()) From 7690247ab43db55e309a357494802c3ec724c0a6 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 18 Mar 2024 20:24:02 +0100 Subject: [PATCH 103/123] Font download: log the exact error --- bookwyrm/apps.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bookwyrm/apps.py b/bookwyrm/apps.py index 5a9f45db5..41b1a17a2 100644 --- a/bookwyrm/apps.py +++ b/bookwyrm/apps.py @@ -19,12 +19,12 @@ def download_file(url, destination): with urllib.request.urlopen(url) as stream: with open(destination, "b+w") as outfile: outfile.write(stream.read()) - except (urllib.error.HTTPError, urllib.error.URLError): - logger.info("Failed to download file %s", url) - except OSError: - logger.info("Couldn't open font file %s for writing", destination) - except: # pylint: disable=bare-except - logger.info("Unknown error in file download") + except (urllib.error.HTTPError, urllib.error.URLError) as err: + logger.error("Failed to download file %s: %s", url, err) + except OSError as err: + logger.error("Couldn't open font file %s for writing: %s", destination, err) + except Exception as err: # pylint:disable=broad-except + logger.error("Unknown error in file download: %s", err) class BookwyrmConfig(AppConfig): From 864304f128fc897348342ba83506441469e8fe53 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 18 Mar 2024 21:01:20 +0100 Subject: [PATCH 104/123] docker-compose.yml: make all bind mounts read only Except dev-tools, since it needs to be able to change the source. --- docker-compose.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4d4037681..6b68d6826 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: networks: - main volumes: - - ./nginx:/etc/nginx/conf.d + - ./nginx:/etc/nginx/conf.d:ro - static_volume:/app/static - media_volume:/app/images db: @@ -26,7 +26,7 @@ services: env_file: .env command: python manage.py runserver 0.0.0.0:8000 volumes: - - .:/app + - .:/app:ro - static_volume:/app/static - media_volume:/app/images depends_on: @@ -41,7 +41,7 @@ services: image: redis:7.2.1 command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT} volumes: - - ./redis.conf:/etc/redis/redis.conf + - ./redis.conf:/etc/redis/redis.conf:ro - redis_activity_data:/data env_file: .env networks: @@ -51,7 +51,7 @@ services: image: redis:7.2.1 command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT} volumes: - - ./redis.conf:/etc/redis/redis.conf + - ./redis.conf:/etc/redis/redis.conf:ro - redis_broker_data:/data env_file: .env networks: @@ -63,9 +63,8 @@ services: networks: - main command: celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority,streams,images,suggested_users,email,connectors,lists,inbox,imports,import_triggered,broadcast,misc - volumes: - - .:/app + - .:/app:ro - static_volume:/app/static - media_volume:/app/images depends_on: @@ -79,7 +78,7 @@ services: - main command: celery -A celerywyrm beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler volumes: - - .:/app + - .:/app:ro - static_volume:/app/static - media_volume:/app/images depends_on: @@ -90,7 +89,7 @@ services: command: celery -A celerywyrm flower --basic_auth=${FLOWER_USER}:${FLOWER_PASSWORD} --url_prefix=flower env_file: .env volumes: - - .:/app + - .:/app:ro networks: - main depends_on: @@ -102,7 +101,7 @@ services: env_file: .env volumes: - /app/dev-tools/ - - .:/app + - .:/app:rw volumes: pgdata: static_volume: From 68cb94daf2e81ed755a4b04f73034e4a3f77d1be Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 18 Mar 2024 21:02:06 +0100 Subject: [PATCH 105/123] docker-compose.yml: don't automatically start dev-tools by assigning profile --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 6b68d6826..9e2cd67ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,6 +102,8 @@ services: volumes: - /app/dev-tools/ - .:/app:rw + profiles: + - tools volumes: pgdata: static_volume: From 4d23edddca5692d7ef66f8b66b88e6f3bc061008 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 18 Mar 2024 21:04:34 +0100 Subject: [PATCH 106/123] Make sure /images/ and /static/ exist now that the bind mount is read only Otherwise the static_volume and media_volume can't be mounted there. --- .gitignore | 1 + images/.gitkeep | 0 static/.gitkeep | 0 3 files changed, 1 insertion(+) create mode 100644 images/.gitkeep create mode 100644 static/.gitkeep diff --git a/.gitignore b/.gitignore index ec2a08f80..755375b34 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ # BookWyrm .env /images/ +/static/ bookwyrm/static/css/bookwyrm.css bookwyrm/static/css/themes/ !bookwyrm/static/css/themes/bookwyrm-*.scss diff --git a/images/.gitkeep b/images/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/static/.gitkeep b/static/.gitkeep new file mode 100644 index 000000000..e69de29bb From 47afe34d9767ad7895b7eb72bc4cd88f99eeb148 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 21:48:21 +0000 Subject: [PATCH 107/123] Bump django from 3.2.24 to 3.2.25 Bumps [django](https://github.com/django/django) from 3.2.24 to 3.2.25. - [Commits](https://github.com/django/django/compare/3.2.24...3.2.25) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c769916a2..3c0406acd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ boto3==1.26.57 bw-file-resubmit==0.6.0rc2 celery==5.3.1 colorthief==0.2.1 -Django==3.2.24 +Django==3.2.25 django-celery-beat==2.5.0 django-compressor==4.4 django-csp==3.7 From f423834bd0d8bd54afa29f56fd60459ac7209b67 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Mon, 18 Mar 2024 20:06:12 +0100 Subject: [PATCH 108/123] Catch the correct exception type from Pillow --- bookwyrm/preview_images.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index aba372abc..995f25bfd 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -1,4 +1,5 @@ """ Generate social media preview images for twitter/mastodon/etc """ + import math import os import textwrap @@ -42,8 +43,8 @@ def get_imagefont(name, size): return ImageFont.truetype(path, size) except KeyError: logger.error("Font %s not found in config", name) - except OSError: - logger.error("Could not load font %s from file", name) + except OSError as err: + logger.error("Could not load font %s from file: %s", name, err) return ImageFont.load_default() @@ -59,7 +60,7 @@ def get_font(weight, size=28): font.set_variation_by_name("Bold") if weight == "regular": font.set_variation_by_name("Regular") - except AttributeError: + except OSError: pass return font From 6a87713f9f4f1c6dd9ad169f967b1335692e203a Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 20 Mar 2024 11:45:12 +0100 Subject: [PATCH 109/123] Recalculate all book search vectors after fixing the author trigger --- .../0191_migrate_search_vec_triggers_to_pgtriggers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py index 5e798b654..03442298f 100644 --- a/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py +++ b/bookwyrm/migrations/0191_migrate_search_vec_triggers_to_pgtriggers.py @@ -67,4 +67,10 @@ class Migration(migrations.Migration): """, reverse_sql=author_search_vector_trigger.sql, ), + migrations.RunSQL( + # Recalculate book search vector for any missed author name changes + # due to bug in JOIN in the old trigger. + sql="UPDATE bookwyrm_book SET search_vector = NULL;", + reverse_sql=migrations.RunSQL.noop, + ), ] From e13e4237f49e1079eb0190898c4b45b53399bcf9 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 20 Mar 2024 12:24:54 +0100 Subject: [PATCH 110/123] black: specify required-version This ensures consistent formatting among different contributors / development setups. https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#required-version --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..292ca8c41 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +required-version = "22" From ab430e020824f08ca15fe3e0da824d9c5865d5c4 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 20 Mar 2024 12:43:14 +0100 Subject: [PATCH 111/123] requirements.txt: add black This way, IDEs can be set up to use the black version from the environment instead of a globally available/bundled black version. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 82b7bc8c8..6b3d838bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,7 @@ setuptools>=65.5.1 # Not a direct dependency, pinned to get a security fix tornado==6.3.3 # Not a direct dependency, pinned to get a security fix # Dev +black==22.* celery-types==0.18.0 django-stubs[compatible-mypy]==4.2.4 mypy==1.5.1 From b5b9eddaf0a8336cf9766c901811022713fb78a6 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 20 Mar 2024 12:46:37 +0100 Subject: [PATCH 112/123] CI: relax black version constraints --- .github/workflows/python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index dcbe05aee..01241b467 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -94,6 +94,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - - uses: psf/black@22.12.0 + - uses: psf/black@stable with: - version: 22.12.0 + version: "22.*" From 682bb3b62fb6ed53daf374376c9bd91f30295809 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Thu, 21 Mar 2024 16:25:29 +0100 Subject: [PATCH 113/123] dev-tools: relax black version constraint --- dev-tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/requirements.txt b/dev-tools/requirements.txt index a69d319ab..3bb771f5a 100644 --- a/dev-tools/requirements.txt +++ b/dev-tools/requirements.txt @@ -1 +1 @@ -black==22.12.0 +black==22.* From c3d25c59c51431cf7d5a2700b842a07410dd8936 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Thu, 21 Mar 2024 16:48:28 +0100 Subject: [PATCH 114/123] Escape search query in generated URLs Otherwise, a query containing '&' or other special characters results in a broken URL. --- bookwyrm/templates/search/book.html | 2 +- bookwyrm/templates/search/layout.html | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bookwyrm/templates/search/book.html b/bookwyrm/templates/search/book.html index 262dcf2f9..b93c96754 100644 --- a/bookwyrm/templates/search/book.html +++ b/bookwyrm/templates/search/book.html @@ -109,7 +109,7 @@

    {% if request.user.is_authenticated %} {% if not remote %} - + {% trans "Load results from other catalogues" %} {% else %} diff --git a/bookwyrm/templates/search/layout.html b/bookwyrm/templates/search/layout.html index 725a4f43f..59ea0304e 100644 --- a/bookwyrm/templates/search/layout.html +++ b/bookwyrm/templates/search/layout.html @@ -41,18 +41,18 @@

    From a914a44fba28defec6fefcbf90deee828176ddf5 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 23 Mar 2024 07:54:54 -0700 Subject: [PATCH 115/123] Removes unnecessary redeclaration of wikidata model field in Author --- bookwyrm/models/author.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index b32d49d54..981e3c0cc 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -18,9 +18,6 @@ class Author(BookDataModel): wikipedia_link = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) - wikidata = fields.CharField( - max_length=255, blank=True, null=True, deduplication_field=True - ) isni = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) From 2de35f3fc7ef1a1418ed4638cce89d2b7dfae999 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 20 Mar 2024 16:14:26 +0100 Subject: [PATCH 116/123] Calculate Author search vector with name and aliases --- .../migrations/0197_author_search_vector.py | 41 +++++++++++++++++++ bookwyrm/models/author.py | 21 +++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 bookwyrm/migrations/0197_author_search_vector.py diff --git a/bookwyrm/migrations/0197_author_search_vector.py b/bookwyrm/migrations/0197_author_search_vector.py new file mode 100644 index 000000000..baa540cc0 --- /dev/null +++ b/bookwyrm/migrations/0197_author_search_vector.py @@ -0,0 +1,41 @@ +# Generated by Django 3.2.25 on 2024-03-20 15:15 + +import django.contrib.postgres.indexes +from django.db import migrations +import pgtrigger.compiler +import pgtrigger.migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0196_merge_pr3134_into_main"), + ] + + operations = [ + migrations.AddIndex( + model_name="author", + index=django.contrib.postgres.indexes.GinIndex( + fields=["search_vector"], name="bookwyrm_au_search__b050a8_gin" + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="author", + trigger=pgtrigger.compiler.Trigger( + name="update_search_vector_on_author_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="new.search_vector := setweight(to_tsvector('simple', new.name), 'A') || setweight(to_tsvector('simple', coalesce(array_to_string(new.aliases, ' '), '')), 'B');RETURN NEW;", + hash="b97919016236d74d0ade51a0769a173ea269da64", + operation='INSERT OR UPDATE OF "name", "aliases", "search_vector"', + pgid="pgtrigger_update_search_vector_on_author_edit_c61cb", + table="bookwyrm_author", + when="BEFORE", + ), + ), + ), + migrations.RunSQL( + # Calculate search vector for all Authors. + sql="UPDATE bookwyrm_author SET search_vector = NULL;", + reverse_sql="UPDATE bookwyrm_author SET search_vector = NULL;", + ), + ] diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 154b00ccb..9c3621c3d 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -3,6 +3,7 @@ import re from typing import Tuple, Any from django.db import models +from django.contrib.postgres.indexes import GinIndex import pgtrigger from bookwyrm import activitypub @@ -71,7 +72,25 @@ class Author(BookDataModel): class Meta: """sets up indexes and triggers""" + # pylint: disable=line-too-long + + indexes = (GinIndex(fields=["search_vector"]),) triggers = [ + pgtrigger.Trigger( + name="update_search_vector_on_author_edit", + when=pgtrigger.Before, + operation=pgtrigger.Insert + | pgtrigger.UpdateOf("name", "aliases", "search_vector"), + func=format_trigger( + """new.search_vector := + -- author name, with priority A + setweight(to_tsvector('simple', new.name), 'A') || + -- author aliases, with priority B + setweight(to_tsvector('simple', coalesce(array_to_string(new.aliases, ' '), '')), 'B'); + RETURN new; + """ + ), + ), pgtrigger.Trigger( name="reset_search_vector_on_author_edit", when=pgtrigger.After, @@ -89,7 +108,7 @@ class Author(BookDataModel): RETURN new; """ ), - ) + ), ] activity_serializer = activitypub.Author From 0795b4d171c135e2832bb81f574150703adc1f3b Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 20 Mar 2024 16:18:55 +0100 Subject: [PATCH 117/123] Include Author aliases in Book search vector --- .../0198_book_search_vector_author_aliases.py | 57 +++++++++++++++++++ bookwyrm/models/author.py | 4 +- bookwyrm/models/book.py | 46 +++++++++------ 3 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 bookwyrm/migrations/0198_book_search_vector_author_aliases.py diff --git a/bookwyrm/migrations/0198_book_search_vector_author_aliases.py b/bookwyrm/migrations/0198_book_search_vector_author_aliases.py new file mode 100644 index 000000000..491cb64bb --- /dev/null +++ b/bookwyrm/migrations/0198_book_search_vector_author_aliases.py @@ -0,0 +1,57 @@ +# Generated by Django 3.2.25 on 2024-03-20 15:52 + +from django.db import migrations +import pgtrigger.compiler +import pgtrigger.migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0197_author_search_vector"), + ] + + operations = [ + pgtrigger.migrations.RemoveTrigger( + model_name="author", + name="reset_search_vector_on_author_edit", + ), + pgtrigger.migrations.RemoveTrigger( + model_name="book", + name="update_search_vector_on_book_edit", + ), + pgtrigger.migrations.AddTrigger( + model_name="author", + trigger=pgtrigger.compiler.Trigger( + name="reset_book_search_vector_on_author_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="WITH updated_books AS (SELECT book_id FROM bookwyrm_book_authors WHERE author_id = new.id ) UPDATE bookwyrm_book SET search_vector = '' FROM updated_books WHERE id = updated_books.book_id;RETURN NEW;", + hash="68422c0f29879c5802b82159dde45297eff53e73", + operation='UPDATE OF "name", "aliases"', + pgid="pgtrigger_reset_book_search_vector_on_author_edit_a50c7", + table="bookwyrm_author", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="book", + trigger=pgtrigger.compiler.Trigger( + name="update_search_vector_on_book_edit", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="WITH author_names AS (SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases FROM bookwyrm_author LEFT JOIN bookwyrm_book_authors ON bookwyrm_author.id = bookwyrm_book_authors.author_id WHERE bookwyrm_book_authors.book_id = new.id ) SELECT setweight(coalesce(nullif(to_tsvector('english', new.title), ''), to_tsvector('simple', new.title)), 'A') || setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') || (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(name_and_aliases), ' '), '')), 'C') FROM author_names) || setweight(to_tsvector('english', coalesce(new.series, '')), 'D') INTO new.search_vector;RETURN NEW;", + hash="9324f5ca76a6f5e63931881d62d11da11f595b2c", + operation='INSERT OR UPDATE OF "title", "subtitle", "series", "search_vector"', + pgid="pgtrigger_update_search_vector_on_book_edit_bec58", + table="bookwyrm_book", + when="BEFORE", + ), + ), + ), + migrations.RunSQL( + # Recalculate search vector for all Books because it now includes + # Author aliases. + sql="UPDATE bookwyrm_book SET search_vector = NULL;", + reverse_sql="UPDATE bookwyrm_book SET search_vector = NULL;", + ), + ] diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 9c3621c3d..9dc3962ad 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -92,9 +92,9 @@ class Author(BookDataModel): ), ), pgtrigger.Trigger( - name="reset_search_vector_on_author_edit", + name="reset_book_search_vector_on_author_edit", when=pgtrigger.After, - operation=pgtrigger.UpdateOf("name"), + operation=pgtrigger.UpdateOf("name", "aliases"), func=format_trigger( """WITH updated_books AS ( SELECT book_id diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index e167e2138..5dba6532f 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -246,24 +246,34 @@ class Book(BookDataModel): operation=pgtrigger.Insert | pgtrigger.UpdateOf("title", "subtitle", "series", "search_vector"), func=format_trigger( - """new.search_vector := - -- title, with priority A (parse in English, default to simple if empty) - setweight(COALESCE(nullif( - to_tsvector('english', new.title), ''), - to_tsvector('simple', new.title)), 'A') || - -- subtitle, with priority B (always in English?) - setweight(to_tsvector('english', COALESCE(new.subtitle, '')), 'B') || - -- list of authors, with priority C (TODO: add aliases?, bookwyrm-social#3063) - (SELECT setweight(to_tsvector('simple', COALESCE(array_to_string(ARRAY_AGG(bookwyrm_author.name), ' '), '')), 'C') - FROM bookwyrm_author - LEFT JOIN bookwyrm_book_authors - ON bookwyrm_author.id = bookwyrm_book_authors.author_id - WHERE bookwyrm_book_authors.book_id = new.id - ) || - --- last: series name, with lowest priority - setweight(to_tsvector('english', COALESCE(new.series, '')), 'D'); - RETURN new; - """ + """ + WITH author_names AS ( + SELECT array_to_string(bookwyrm_author.name || bookwyrm_author.aliases, ' ') AS name_and_aliases + FROM bookwyrm_author + LEFT JOIN bookwyrm_book_authors + ON bookwyrm_author.id = bookwyrm_book_authors.author_id + WHERE bookwyrm_book_authors.book_id = new.id + ) + SELECT + -- title, with priority A (parse in English, default to simple if empty) + setweight(COALESCE(nullif( + to_tsvector('english', new.title), ''), + to_tsvector('simple', new.title)), 'A') || + + -- subtitle, with priority B (always in English?) + setweight(to_tsvector('english', COALESCE(new.subtitle, '')), 'B') || + + -- list of authors names and aliases (with priority C) + (SELECT setweight(to_tsvector('simple', COALESCE(array_to_string(ARRAY_AGG(name_and_aliases), ' '), '')), 'C') + FROM author_names + ) || + + --- last: series name, with lowest priority + setweight(to_tsvector('english', COALESCE(new.series, '')), 'D') + + INTO new.search_vector; + RETURN new; + """ ), ) ] From 36222afa7995e19e6daf6cb4523039d140adf4af Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Tue, 19 Mar 2024 13:12:45 +0100 Subject: [PATCH 118/123] Switch author search from TrigramSimilarity to SearchQuery --- bookwyrm/views/search.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/bookwyrm/views/search.py b/bookwyrm/views/search.py index e1598056f..13695a7d4 100644 --- a/bookwyrm/views/search.py +++ b/bookwyrm/views/search.py @@ -2,8 +2,9 @@ import re -from django.contrib.postgres.search import TrigramSimilarity +from django.contrib.postgres.search import TrigramSimilarity, SearchRank, SearchQuery from django.core.paginator import Paginator +from django.db.models import F from django.db.models.functions import Greatest from django.http import JsonResponse from django.template.response import TemplateResponse @@ -94,26 +95,28 @@ def book_search(request): def author_search(request): """search for an author""" - query = request.GET.get("q") - query = query.strip() - data = {"type": "author", "query": query} + query = request.GET.get("q").strip() + search_query = SearchQuery(query, config="simple") + min_confidence = 0 results = ( - models.Author.objects.annotate( - similarity=TrigramSimilarity("name", query), - ) - .filter( - similarity__gt=0.1, - ) - .order_by("-similarity") + models.Author.objects.filter(search_vector=search_query) + .annotate(rank=SearchRank(F("search_vector"), search_query)) + .filter(rank__gt=min_confidence) + .order_by("-rank") ) paginated = Paginator(results, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) - data["results"] = page - data["page_range"] = paginated.get_elided_page_range( - page.number, on_each_side=2, on_ends=1 - ) + + data = { + "type": "author", + "query": query, + "results": page, + "page_range": paginated.get_elided_page_range( + page.number, on_each_side=2, on_ends=1 + ), + } return TemplateResponse(request, "search/author.html", data) From 769d9726e5cfb69ab8dcaa173dc1bcb062a29db0 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 22 Mar 2024 21:43:28 +0100 Subject: [PATCH 119/123] Add book search test cases for author aliases --- bookwyrm/tests/test_book_search.py | 57 ++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index 0721cb142..b8c1ee1d3 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -14,6 +14,13 @@ class BookSearch(TestCase): @classmethod def setUpTestData(self): # pylint: disable=bad-classmethod-argument """we need basic test data and mocks""" + self.first_author = models.Author.objects.create( + name="Author One", aliases=["The First"] + ) + self.second_author = models.Author.objects.create( + name="Author Two", aliases=["The Second"] + ) + self.work = models.Work.objects.create(title="Example Work") self.first_edition = models.Edition.objects.create( @@ -23,6 +30,8 @@ class BookSearch(TestCase): physical_format="Paperback", published_date=datetime.datetime(2019, 4, 9, 0, 0, tzinfo=timezone.utc), ) + self.first_edition.authors.add(self.first_author) + self.second_edition = models.Edition.objects.create( title="Another Edition", parent_work=self.work, @@ -30,19 +39,34 @@ class BookSearch(TestCase): openlibrary_key="hello", pages=150, ) + self.second_edition.authors.add(self.first_author) + self.second_edition.authors.add(self.second_author) + self.third_edition = models.Edition.objects.create( title="Another Edition with annoying ISBN", parent_work=self.work, isbn_10="022222222X", ) + self.third_edition.authors.add(self.first_author) + self.third_edition.authors.add(self.second_author) def test_search(self): """search for a book in the db""" - # title/author + # title results = book_search.search("Example") self.assertEqual(len(results), 1) self.assertEqual(results[0], self.first_edition) + # author + results = book_search.search("One") + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.first_edition) + + # author alias + results = book_search.search("First") + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.first_edition) + # isbn results = book_search.search("0000000000") self.assertEqual(len(results), 1) @@ -155,8 +179,17 @@ class SearchVectorTest(TestCase): """search vector with subtitle and series""" # for a book like this we call `to_tsvector("Book Long Mary Bunch")`, hence the # indexes in the search vector. (priority "D" is the default, and never shown.) - book = self._create_book("Book", "Mary", subtitle="Long", series="Bunch") - self.assertEqual(book.search_vector, "'book':1A 'bunch':4 'long':2B 'mary':3C") + book = self._create_book( + "Book", + "Mary", + subtitle="Long", + series="Bunch", + author_alias=["Maria", "Mary Ann"], + ) + self.assertEqual( + book.search_vector, + "'ann':6C 'book':1A 'bunch':7 'long':2B 'maria':4C 'mary':3C,5C", + ) def test_search_vector_parse_book(self): """book parts are parsed in english""" @@ -170,8 +203,8 @@ class SearchVectorTest(TestCase): def test_search_vector_parse_author(self): """author name is not stem'd or affected by stop words""" - book = self._create_book("Writing", "Writes") - self.assertEqual(book.search_vector, "'write':1A 'writes':2C") + book = self._create_book("Writing", "Writes", author_alias=["Reads"]) + self.assertEqual(book.search_vector, "'reads':3C 'write':1A 'writes':2C") book = self._create_book("She Is Writing", "She Writes") self.assertEqual(book.search_vector, "'she':4C 'write':3A 'writes':5C") @@ -218,6 +251,13 @@ class SearchVectorTest(TestCase): book.refresh_from_db() self.assertEqual(book.search_vector, "'goodby':3A 'jeremy':4C 'long':2A") + author.aliases = ["Example"] + author.save(broadcast=False) + book.refresh_from_db() + self.assertEqual( + book.search_vector, "'example':5C 'goodby':3A 'jeremy':4C 'long':2A" + ) + def test_search_vector_on_author_delete(self): """update search when an author is deleted""" book = self._create_book("The Long Goodbye", "The Rays") @@ -274,7 +314,7 @@ class SearchVectorUpdates(TestCase): def setUp(self): """we need basic test data and mocks""" self.work = models.Work.objects.create(title="This Work") - self.author = models.Author.objects.create(name="Name") + self.author = models.Author.objects.create(name="Name", aliases=["Alias"]) self.edition = models.Edition.objects.create( title="First Edition of Work", subtitle="Some Extra Words Are Good", @@ -363,13 +403,18 @@ class SearchVectorUpdates(TestCase): def test_search_after_updated_author_name(self): """book found under new author name""" self.assertEqual(self.edition, self._search_first("Name")) + self.assertEqual(self.edition, self._search_first("Alias")) self.assertFalse(self._search("Identifier")) + self.assertFalse(self._search("Another")) self.author.name = "Identifier" + self.author.aliases = ["Another"] self.author.save(broadcast=False) self.assertFalse(self._search("Name")) + self.assertFalse(self._search("Aliases")) self.assertEqual(self.edition, self._search_first("Identifier")) + self.assertEqual(self.edition, self._search_first("Another")) self.assertEqual(self.edition, self._search_first("Work")) def _search_first(self, query): From b8995bd4b1bbb0f26eb0a6751fec1a8622389223 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 22 Mar 2024 22:06:19 +0100 Subject: [PATCH 120/123] Add tests for author search --- bookwyrm/tests/test_author_search.py | 87 ++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 bookwyrm/tests/test_author_search.py diff --git a/bookwyrm/tests/test_author_search.py b/bookwyrm/tests/test_author_search.py new file mode 100644 index 000000000..e6b20a2c6 --- /dev/null +++ b/bookwyrm/tests/test_author_search.py @@ -0,0 +1,87 @@ +""" test searching for authors """ +from django.test import TestCase + +from django.contrib.postgres.search import SearchRank, SearchQuery +from django.db.models import F + +from bookwyrm import models + + +class AuthorSearch(TestCase): + """look for some authors""" + + @classmethod + def setUpTestData(cls): + """we need basic test data and mocks""" + cls.bob = models.Author.objects.create( + name="Bob", aliases=["Robertus", "Alice"] + ) + cls.alice = models.Author.objects.create(name="Alice") + + def test_search(self): + """search for an author in the db""" + results = self._search("Bob") + self.assertEqual(len(results), 1) + self.assertEqual(results[0], self.bob) + + def test_alias_priority(self): + """aliases should be included, with lower priority than name""" + results = self._search("Alice") + self.assertEqual(len(results), 2) + self.assertEqual(results[0], self.alice) + + def _search_first(self, query): + """wrapper around search_title_author""" + return self._search(query, return_first=True) + + @staticmethod + def _search(query, *, return_first=False): + """author search""" + search_query = SearchQuery(query, config="simple") + min_confidence = 0 + + results = ( + models.Author.objects.filter(search_vector=search_query) + .annotate(rank=SearchRank(F("search_vector"), search_query)) + .filter(rank__gt=min_confidence) + .order_by("-rank") + ) + if return_first: + return results.first() + return results + + +class SearchVectorTest(TestCase): + """check search_vector is computed correctly""" + + def test_search_vector_simple(self): + """simplest search vector""" + author = self._create_author("Mary") + self.assertEqual(author.search_vector, "'mary':1A") + + def test_search_vector_aliases(self): + """author aliases should be included with lower priority""" + author = self._create_author("Mary", aliases=["Maria", "Example"]) + self.assertEqual(author.search_vector, "'example':3B 'maria':2B 'mary':1A") + + def test_search_vector_parse_author(self): + """author name and alias is not stem'd or affected by stop words""" + author = self._create_author("Writes", aliases=["Reads"]) + self.assertEqual(author.search_vector, "'reads':2B 'writes':1A") + + def test_search_vector_on_update(self): + """make sure that search_vector is being set correctly on edit""" + author = self._create_author("Mary") + self.assertEqual(author.search_vector, "'mary':1A") + + author.name = "Example" + author.save(broadcast=False) + author.refresh_from_db() + self.assertEqual(author.search_vector, "'example':1A") + + @staticmethod + def _create_author(name, /, *, aliases=None): + """quickly create an author""" + author = models.Author.objects.create(name=name, aliases=aliases or []) + author.refresh_from_db() + return author From 2d2ccd51df4801d2219ad46c2d7bff99d9afc6c3 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 23 Mar 2024 19:35:24 +0100 Subject: [PATCH 121/123] Factor out book series info into separate template --- bookwyrm/templates/snippets/book_series.html | 3 +++ bookwyrm/templates/snippets/book_titleby.html | 8 ++------ bookwyrm/templates/snippets/status/headers/comment.html | 4 +--- bookwyrm/templates/snippets/status/headers/quotation.html | 4 +--- bookwyrm/templates/snippets/status/headers/read.html | 4 +--- bookwyrm/templates/snippets/status/headers/reading.html | 4 +--- bookwyrm/templates/snippets/status/headers/review.html | 4 +--- .../snippets/status/headers/stopped_reading.html | 4 +--- bookwyrm/templates/snippets/status/headers/to_read.html | 4 +--- 9 files changed, 12 insertions(+), 27 deletions(-) create mode 100644 bookwyrm/templates/snippets/book_series.html diff --git a/bookwyrm/templates/snippets/book_series.html b/bookwyrm/templates/snippets/book_series.html new file mode 100644 index 000000000..fd1343604 --- /dev/null +++ b/bookwyrm/templates/snippets/book_series.html @@ -0,0 +1,3 @@ +{% if book.series %} +({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) +{% endif %} diff --git a/bookwyrm/templates/snippets/book_titleby.html b/bookwyrm/templates/snippets/book_titleby.html index 35e0c4c45..518d96389 100644 --- a/bookwyrm/templates/snippets/book_titleby.html +++ b/bookwyrm/templates/snippets/book_titleby.html @@ -9,8 +9,7 @@ {% if book.authors.exists %} {% blocktrans trimmed with path=book.local_path title=book|book_title %} -{{ title }} - +{{ title }} by {% endblocktrans %} {% include 'snippets/authors.html' with book=book limit=3 %} @@ -18,10 +17,7 @@ by {{ book|book_title }} {% endif %} -{% if book.series %} -({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) -{% endif %} - +{% include 'snippets/book_series.html' with book=book %} {% endcache %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/comment.html b/bookwyrm/templates/snippets/status/headers/comment.html index b7750d3df..72bf5b255 100644 --- a/bookwyrm/templates/snippets/status/headers/comment.html +++ b/bookwyrm/templates/snippets/status/headers/comment.html @@ -18,8 +18,6 @@ commented on {{ book }} {% endif %} -{% if book.series %} -({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) -{% endif %} +{% include 'snippets/book_series.html' with book=book %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/quotation.html b/bookwyrm/templates/snippets/status/headers/quotation.html index 3ddd20291..ce0ea5923 100644 --- a/bookwyrm/templates/snippets/status/headers/quotation.html +++ b/bookwyrm/templates/snippets/status/headers/quotation.html @@ -18,8 +18,6 @@ quoted {{ book }} {% endif %} -{% if book.series %} -({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) -{% endif %} +{% include 'snippets/book_series.html' with book=book %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/read.html b/bookwyrm/templates/snippets/status/headers/read.html index 8c323dac6..5a237a836 100644 --- a/bookwyrm/templates/snippets/status/headers/read.html +++ b/bookwyrm/templates/snippets/status/headers/read.html @@ -20,8 +20,6 @@ finished reading {{ book }} {% endif %} -{% if book.series %} -({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) -{% endif %} +{% include 'snippets/book_series.html' with book=book %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/reading.html b/bookwyrm/templates/snippets/status/headers/reading.html index 73e96ebbc..44c604620 100644 --- a/bookwyrm/templates/snippets/status/headers/reading.html +++ b/bookwyrm/templates/snippets/status/headers/reading.html @@ -20,8 +20,6 @@ started reading {{ book }} {% endif %} -{% if book.series %} -({{book.series}} {%if book.series_number %}#{{book.series_number}}) {% endif %} -{% endif %} +{% include 'snippets/book_series.html' with book=book %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/review.html b/bookwyrm/templates/snippets/status/headers/review.html index 0bdcd29b1..95a6fc403 100644 --- a/bookwyrm/templates/snippets/status/headers/review.html +++ b/bookwyrm/templates/snippets/status/headers/review.html @@ -18,8 +18,6 @@ reviewed {{ book }} {% endif %} -{% if book.series %} -({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) -{% endif %} +{% include 'snippets/book_series.html' with book=book %} {% endwith %} diff --git a/bookwyrm/templates/snippets/status/headers/stopped_reading.html b/bookwyrm/templates/snippets/status/headers/stopped_reading.html index a00be181c..09f2e4264 100644 --- a/bookwyrm/templates/snippets/status/headers/stopped_reading.html +++ b/bookwyrm/templates/snippets/status/headers/stopped_reading.html @@ -20,9 +20,7 @@ stopped reading {{ book }} {% endif %} -{% if book.series %} -({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) -{% endif %} +{% include 'snippets/book_series.html' with book=book %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/status/headers/to_read.html b/bookwyrm/templates/snippets/status/headers/to_read.html index 4d1b2ab1b..c775c33a7 100644 --- a/bookwyrm/templates/snippets/status/headers/to_read.html +++ b/bookwyrm/templates/snippets/status/headers/to_read.html @@ -20,8 +20,6 @@ wants to read {{ book }} {% endif %} -{% if book.series %} -({{book.series}}{%if book.series_number %}, #{{book.series_number}}{% endif %}) -{% endif %} +{% include 'snippets/book_series.html' with book=book %} {% endspaceless %} From 592914dc91aad1a2a96c90321eef1006a9d4a060 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sat, 23 Mar 2024 19:51:20 +0100 Subject: [PATCH 122/123] Render series number with comma and outside of link on book page --- bookwyrm/templates/book/book.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bookwyrm/templates/book/book.html b/bookwyrm/templates/book/book.html index 4c345832e..83500a54b 100644 --- a/bookwyrm/templates/book/book.html +++ b/bookwyrm/templates/book/book.html @@ -45,18 +45,22 @@ {% endif %} {% if book.series %} - + {% spaceless %} {% if book.authors.exists %} {% endif %} + {% if book.series_number %} + , # + {{ book.series_number }} + {% endif %} + {% endspaceless %} {% endif %}

    {% endif %} From 7c2fa746ae3c1b0560c2a4411237a688ee513ce5 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Sun, 24 Mar 2024 11:23:23 +0100 Subject: [PATCH 123/123] Revert "docker-compose.yml: make all bind mounts read only" This reverts commit 864304f128fc897348342ba83506441469e8fe53. --- docker-compose.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 034cc2677..71a844ba2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: networks: - main volumes: - - ./nginx:/etc/nginx/conf.d:ro + - ./nginx:/etc/nginx/conf.d - static_volume:/app/static - media_volume:/app/images db: @@ -26,7 +26,7 @@ services: env_file: .env command: python manage.py runserver 0.0.0.0:8000 volumes: - - .:/app:ro + - .:/app - static_volume:/app/static - media_volume:/app/images depends_on: @@ -41,7 +41,7 @@ services: image: redis:7.2.1 command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT} volumes: - - ./redis.conf:/etc/redis/redis.conf:ro + - ./redis.conf:/etc/redis/redis.conf - redis_activity_data:/data env_file: .env networks: @@ -51,7 +51,7 @@ services: image: redis:7.2.1 command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT} volumes: - - ./redis.conf:/etc/redis/redis.conf:ro + - ./redis.conf:/etc/redis/redis.conf - redis_broker_data:/data env_file: .env networks: @@ -64,7 +64,7 @@ services: - main command: celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority,streams,images,suggested_users,email,connectors,lists,inbox,imports,import_triggered,broadcast,misc volumes: - - .:/app:ro + - .:/app - static_volume:/app/static - media_volume:/app/images depends_on: @@ -78,7 +78,7 @@ services: - main command: celery -A celerywyrm beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler volumes: - - .:/app:ro + - .:/app - static_volume:/app/static - media_volume:/app/images depends_on: @@ -89,7 +89,7 @@ services: command: celery -A celerywyrm flower --basic_auth=${FLOWER_USER}:${FLOWER_PASSWORD} --url_prefix=flower env_file: .env volumes: - - .:/app:ro + - .:/app - static_volume:/app/static networks: - main @@ -102,7 +102,7 @@ services: env_file: .env volumes: - /app/dev-tools/ - - .:/app:rw + - .:/app profiles: - tools volumes: