From d93da4e86d771f88d654339b3563a852dc69c767 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Mon, 27 Nov 2023 15:03:59 -0800 Subject: [PATCH 02/56] Checkpoint --- bookwyrm/book_search.py | 20 ++++++++----- bookwyrm/settings.py | 2 +- .../templates/shelf/search_my_books_form.html | 25 ++++++++++++++++ bookwyrm/templates/shelf/shelf.html | 1 + bookwyrm/tests/views/test_search.py | 2 +- bookwyrm/views/search.py | 6 ++-- bookwyrm/views/shelf/shelf.py | 29 +++++++++++++------ docker-compose.yml | 2 ++ 8 files changed, 66 insertions(+), 21 deletions(-) create mode 100644 bookwyrm/templates/shelf/search_my_books_form.html diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index ceb228f40..6ab33fd5a 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -43,6 +43,7 @@ def search( min_confidence: float = 0, filters: Optional[list[Any]] = None, return_first: bool = False, + books = None ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: """search your local database""" filters = filters or [] @@ -54,17 +55,16 @@ def search( # first, try searching unique identifiers # unique identifiers never have spaces, title/author usually do if not " " in query: - results = search_identifiers(query, *filters, return_first=return_first) + results = search_identifiers(query, *filters, return_first=return_first, books=books) # if there were no identifier results... if not results: # then try searching title/author results = search_title_author( - query, min_confidence, *filters, return_first=return_first + query, min_confidence, *filters, return_first=return_first, books=books ) return results - def isbn_search(query): """search your local database""" if not query: @@ -98,8 +98,9 @@ def format_search_result(search_result): def search_identifiers( - query, *filters, return_first=False + query, *filters, return_first=False, books=None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: + books = books or models.Edition """tries remote_id, isbn; defined as dedupe fields on the model""" if connectors.maybe_isbn(query): # Oh did you think the 'S' in ISBN stood for 'standard'? @@ -111,7 +112,7 @@ def search_identifiers( for f in models.Edition._meta.get_fields() if hasattr(f, "deduplication_field") and f.deduplication_field ] - results = models.Edition.objects.filter( + results = books.filter( *filters, reduce(operator.or_, (Q(**f) for f in or_filters)) ).distinct() @@ -121,12 +122,17 @@ def search_identifiers( def search_title_author( - query, min_confidence, *filters, return_first=False + query, + min_confidence, + *filters, + return_first=False, + books=None, ) -> QuerySet[models.Edition]: """searches for title and author""" + books = books or models.Edition.objects query = SearchQuery(query, config="simple") | SearchQuery(query, config="english") results = ( - models.Edition.objects.filter(*filters, search_vector=query) + books.filter(*filters, search_vector=query) .annotate(rank=SearchRank(F("search_vector"), query)) .filter(rank__gt=min_confidence) .order_by("-rank") diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 4cecc4df6..49e4e2116 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -140,7 +140,7 @@ TEMPLATES = [ }, ] -LOG_LEVEL = env("LOG_LEVEL", "INFO").upper() +LOG_LEVEL = env("LOG_LEVEL", "DEBUG").upper() # Override aspects of the default handler to our taste # See https://docs.djangoproject.com/en/3.2/topics/logging/#default-logging-configuration # for a reference to the defaults we're overriding diff --git a/bookwyrm/templates/shelf/search_my_books_form.html b/bookwyrm/templates/shelf/search_my_books_form.html new file mode 100644 index 000000000..ca9cd2991 --- /dev/null +++ b/bookwyrm/templates/shelf/search_my_books_form.html @@ -0,0 +1,25 @@ +{% load i18n %} +{% load utilities %} + +
+
+
+ {% trans "Search for a book" as my_shelves_search_placeholder %} + +
+
+ +
+
+ +
+
+
diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index a2410ef95..b84ae185f 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -123,6 +123,7 @@ {% endif %} {% endwith %} + {% include 'shelf/search_my_books_form.html' with user=user query=query %} {% if is_self and shelf.id %} diff --git a/bookwyrm/tests/views/test_search.py b/bookwyrm/tests/views/test_search.py index 28f8268e3..021d502f0 100644 --- a/bookwyrm/tests/views/test_search.py +++ b/bookwyrm/tests/views/test_search.py @@ -9,7 +9,7 @@ from django.test import TestCase from django.test.client import RequestFactory from bookwyrm import models, views -from bookwyrm.book_search import SearchResult +from bookwyrm.book_search import SearchResult, search from bookwyrm.settings import DOMAIN from bookwyrm.tests.validate_html import validate_html diff --git a/bookwyrm/views/search.py b/bookwyrm/views/search.py index 743f33f59..f8eddd1a6 100644 --- a/bookwyrm/views/search.py +++ b/bookwyrm/views/search.py @@ -51,7 +51,7 @@ class Search(View): def api_book_search(request): """Return books via API response""" query = request.GET.get("q") - query = isbn_check(query) + query = isbn_check_and_format(query) min_confidence = request.GET.get("min_confidence", 0) # only return local book results via json so we don't cascade book_results = search(query, min_confidence=min_confidence) @@ -64,7 +64,7 @@ def book_search(request): """the real business is elsewhere""" query = request.GET.get("q") # check if query is isbn - query = isbn_check(query) + query = isbn_check_and_format(query) min_confidence = request.GET.get("min_confidence", 0) search_remote = request.GET.get("remote", False) and request.user.is_authenticated @@ -159,7 +159,7 @@ def list_search(request): return TemplateResponse(request, "search/list.html", data) -def isbn_check(query): +def isbn_check_and_format(query): """isbn10 or isbn13 check, if so remove separators""" if query: su_num = re.sub(r"(?<=\d)\D(?=\d|[xX])", "", query) diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index dbbcc2d3a..600dfb731 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -1,7 +1,7 @@ """ shelf views """ from collections import namedtuple -from django.db.models import OuterRef, Subquery, F, Max +from django.db.models import OuterRef, Subquery, F, Max, QuerySet from django.contrib.auth.decorators import login_required from django.core.paginator import Paginator from django.http import HttpResponseBadRequest @@ -15,6 +15,10 @@ from bookwyrm import forms, models from bookwyrm.activitypub import ActivitypubResponse from bookwyrm.settings import PAGE_LENGTH from bookwyrm.views.helpers import is_api_request, get_user_from_username +from bookwyrm.book_search import search + +import logging +logger = logging.getLogger(__name__) # pylint: disable=no-self-use @@ -32,6 +36,8 @@ class Shelf(View): else: shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all() + shelves_search_query = request.GET.get("shelves_q") + # get the shelf and make sure the logged in user should be able to see it if shelf_identifier: shelf = get_object_or_404(user.shelf_set, identifier=shelf_identifier) @@ -42,14 +48,17 @@ class Shelf(View): FakeShelf = namedtuple( "Shelf", ("identifier", "name", "user", "books", "privacy") ) - books = ( - models.Edition.viewer_aware_objects(request.user) - .filter( - # privacy is ensured because the shelves are already filtered above - shelfbook__shelf__in=shelves - ) - .distinct() - ) + if shelves_search_query: + logger.debug("AAAAAAAAAAAA") + all_books = models.Edition.viewer_aware_objects(request.user).filter( + # privacy is ensured because the shelves are already filtered above + shelfbook__shelf__in=shelves + ).distinct() + books = search(shelves_search_query, books=all_books) + else: + logger.debug("BBBBBBBBB") + books = shelf.books + shelf = FakeShelf("all", _("All books"), user, books, "public") if is_api_request(request) and shelf_identifier: @@ -103,6 +112,8 @@ class Shelf(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), + "has_shelves_query": bool(shelves_search_query), + "shelves_search_query": shelves_search_query } return TemplateResponse(request, "shelf/shelf.html", data) diff --git a/docker-compose.yml b/docker-compose.yml index 4d4037681..bb0acbdd1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,8 @@ services: - pgdata:/var/lib/postgresql/data networks: - main + ports: + - "5432:5432" web: build: . env_file: .env From b27ed847d575a7d504b04792e45bd0c73a2dffa4 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 5 Dec 2023 16:36:58 -0800 Subject: [PATCH 03/56] Fixes result set passed to template --- bookwyrm/book_search.py | 6 ++++-- bookwyrm/settings.py | 2 +- bookwyrm/views/shelf/shelf.py | 18 +++++++++++------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index 6ab33fd5a..db7a51426 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -14,6 +14,7 @@ from bookwyrm import connectors from bookwyrm.settings import MEDIA_FULL_URL + @overload def search( query: str, @@ -43,7 +44,7 @@ def search( min_confidence: float = 0, filters: Optional[list[Any]] = None, return_first: bool = False, - books = None + books: Optional[QuerySet[models.Edition]] = None ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: """search your local database""" filters = filters or [] @@ -59,6 +60,7 @@ def search( # if there were no identifier results... if not results: + print('SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS') # then try searching title/author results = search_title_author( query, min_confidence, *filters, return_first=return_first, books=books @@ -100,7 +102,7 @@ def format_search_result(search_result): def search_identifiers( query, *filters, return_first=False, books=None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: - books = books or models.Edition + books = books or models.Edition.objects """tries remote_id, isbn; defined as dedupe fields on the model""" if connectors.maybe_isbn(query): # Oh did you think the 'S' in ISBN stood for 'standard'? diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 49e4e2116..c5f5f52f4 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -194,7 +194,7 @@ LOGGING = { # Add a bookwyrm-specific logger "bookwyrm": { "handlers": ["console"], - "level": LOG_LEVEL, + "level": DEBUG, }, }, } diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 600dfb731..159cc1541 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -43,21 +43,23 @@ class Shelf(View): shelf = get_object_or_404(user.shelf_set, identifier=shelf_identifier) shelf.raise_visible_to_user(request.user) books = shelf.books + if shelves_search_query: + books = search(shelves_search_query, books=books) else: # this is a constructed "all books" view, with a fake "shelf" obj FakeShelf = namedtuple( "Shelf", ("identifier", "name", "user", "books", "privacy") ) - if shelves_search_query: - logger.debug("AAAAAAAAAAAA") - all_books = models.Edition.viewer_aware_objects(request.user).filter( + + books = models.Edition.viewer_aware_objects(request.user).filter( # privacy is ensured because the shelves are already filtered above shelfbook__shelf__in=shelves ).distinct() - books = search(shelves_search_query, books=all_books) - else: - logger.debug("BBBBBBBBB") - books = shelf.books + + # TODO: [COMMENT] + if shelves_search_query: + books = search(shelves_search_query, books=books) + books = models.Edition.objects.filter(pk__in=books) shelf = FakeShelf("all", _("All books"), user, books, "public") @@ -81,6 +83,8 @@ class Shelf(View): "start_date" ) + # import pdb; pdb.set_trace() + books = books.annotate(shelved_date=Max("shelfbook__shelved_date")) books = books.annotate( rating=Subquery(reviews.values("rating")[:1]), From 979162da10c68cae5ad694d5b3f74d96d84e9acf Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 5 Dec 2023 19:33:59 -0800 Subject: [PATCH 04/56] Uses filters, fixes for any shelf --- bookwyrm/book_search.py | 1 - .../templates/shelf/search_filter_field.html | 9 +++++++ bookwyrm/templates/shelf/search_filters.html | 5 ++++ .../templates/shelf/search_my_books_form.html | 25 ------------------- bookwyrm/templates/shelf/shelf.html | 5 +++- bookwyrm/views/shelf/shelf.py | 13 +++------- 6 files changed, 21 insertions(+), 37 deletions(-) create mode 100644 bookwyrm/templates/shelf/search_filter_field.html create mode 100644 bookwyrm/templates/shelf/search_filters.html delete mode 100644 bookwyrm/templates/shelf/search_my_books_form.html diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index db7a51426..b3e66cdd6 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -60,7 +60,6 @@ def search( # if there were no identifier results... if not results: - print('SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS') # then try searching title/author results = search_title_author( query, min_confidence, *filters, return_first=return_first, books=books diff --git a/bookwyrm/templates/shelf/search_filter_field.html b/bookwyrm/templates/shelf/search_filter_field.html new file mode 100644 index 000000000..591e6aaa9 --- /dev/null +++ b/bookwyrm/templates/shelf/search_filter_field.html @@ -0,0 +1,9 @@ +{% extends 'snippets/filters_panel/filter_field.html' %} +{% load i18n %} + +{% block filter %} +
+ + +
+{% endblock %} diff --git a/bookwyrm/templates/shelf/search_filters.html b/bookwyrm/templates/shelf/search_filters.html new file mode 100644 index 000000000..840eec57e --- /dev/null +++ b/bookwyrm/templates/shelf/search_filters.html @@ -0,0 +1,5 @@ +{% extends 'snippets/filters_panel/filters_panel.html' %} + +{% block filter_fields %} + {% include 'shelf/search_filter_field.html' %} +{% endblock %} diff --git a/bookwyrm/templates/shelf/search_my_books_form.html b/bookwyrm/templates/shelf/search_my_books_form.html deleted file mode 100644 index ca9cd2991..000000000 --- a/bookwyrm/templates/shelf/search_my_books_form.html +++ /dev/null @@ -1,25 +0,0 @@ -{% load i18n %} -{% load utilities %} - -
-
-
- {% trans "Search for a book" as my_shelves_search_placeholder %} - -
-
- -
-
- -
-
-
diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index b84ae185f..ea5d2343c 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -121,10 +121,13 @@ {% endblocktrans %} {% endif %} + {% endif %} + {% endwith %} - {% include 'shelf/search_my_books_form.html' with user=user query=query %} + + {% include 'shelf/search_filters.html' with user=user query=query %} {% if is_self and shelf.id %}
diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 159cc1541..be205597b 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -43,8 +43,6 @@ class Shelf(View): shelf = get_object_or_404(user.shelf_set, identifier=shelf_identifier) shelf.raise_visible_to_user(request.user) books = shelf.books - if shelves_search_query: - books = search(shelves_search_query, books=books) else: # this is a constructed "all books" view, with a fake "shelf" obj FakeShelf = namedtuple( @@ -56,11 +54,6 @@ class Shelf(View): shelfbook__shelf__in=shelves ).distinct() - # TODO: [COMMENT] - if shelves_search_query: - books = search(shelves_search_query, books=books) - books = models.Edition.objects.filter(pk__in=books) - shelf = FakeShelf("all", _("All books"), user, books, "public") if is_api_request(request) and shelf_identifier: @@ -83,8 +76,6 @@ class Shelf(View): "start_date" ) - # import pdb; pdb.set_trace() - books = books.annotate(shelved_date=Max("shelfbook__shelved_date")) books = books.annotate( rating=Subquery(reviews.values("rating")[:1]), @@ -99,6 +90,9 @@ class Shelf(View): books = sort_books(books, request.GET.get("sort")) + if shelves_search_query: + books = search(shelves_search_query, books=books) + paginated = Paginator( books, PAGE_LENGTH, @@ -116,7 +110,6 @@ class Shelf(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), - "has_shelves_query": bool(shelves_search_query), "shelves_search_query": shelves_search_query } From c65e165aeb1d36921da1ff110c695c1bfa890f5a Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 5 Dec 2023 19:37:29 -0800 Subject: [PATCH 05/56] Hides filter if shelf empty --- bookwyrm/templates/shelf/search_filter_field.html | 2 +- bookwyrm/templates/shelf/shelf.html | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bookwyrm/templates/shelf/search_filter_field.html b/bookwyrm/templates/shelf/search_filter_field.html index 591e6aaa9..5641bae85 100644 --- a/bookwyrm/templates/shelf/search_filter_field.html +++ b/bookwyrm/templates/shelf/search_filter_field.html @@ -4,6 +4,6 @@ {% block filter %}
- +
{% endblock %} diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index ea5d2343c..4fb78ba6f 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -127,7 +127,10 @@ {% endwith %} - {% include 'shelf/search_filters.html' with user=user query=query %} + {% if books|length > 0 %} + {% include 'shelf/search_filters.html' with user=user query=query %} + {% endif %} +
{% if is_self and shelf.id %}
From 0f6e567b21b9cfbae86c7bfdff82eb0b3e673cc1 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 5 Dec 2023 19:49:38 -0800 Subject: [PATCH 06/56] Clean up --- bookwyrm/settings.py | 4 ++-- bookwyrm/templates/shelf/shelf.html | 3 --- bookwyrm/tests/views/test_search.py | 2 +- bookwyrm/views/shelf/shelf.py | 4 ---- docker-compose.yml | 2 -- 5 files changed, 3 insertions(+), 12 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index c5f5f52f4..4cecc4df6 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -140,7 +140,7 @@ TEMPLATES = [ }, ] -LOG_LEVEL = env("LOG_LEVEL", "DEBUG").upper() +LOG_LEVEL = env("LOG_LEVEL", "INFO").upper() # Override aspects of the default handler to our taste # See https://docs.djangoproject.com/en/3.2/topics/logging/#default-logging-configuration # for a reference to the defaults we're overriding @@ -194,7 +194,7 @@ LOGGING = { # Add a bookwyrm-specific logger "bookwyrm": { "handlers": ["console"], - "level": DEBUG, + "level": LOG_LEVEL, }, }, } diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index 4fb78ba6f..14b748e54 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -121,11 +121,8 @@ {% endblocktrans %} {% endif %} - {% endif %} - {% endwith %} - {% if books|length > 0 %} {% include 'shelf/search_filters.html' with user=user query=query %} diff --git a/bookwyrm/tests/views/test_search.py b/bookwyrm/tests/views/test_search.py index 021d502f0..28f8268e3 100644 --- a/bookwyrm/tests/views/test_search.py +++ b/bookwyrm/tests/views/test_search.py @@ -9,7 +9,7 @@ from django.test import TestCase from django.test.client import RequestFactory from bookwyrm import models, views -from bookwyrm.book_search import SearchResult, search +from bookwyrm.book_search import SearchResult from bookwyrm.settings import DOMAIN from bookwyrm.tests.validate_html import validate_html diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index be205597b..aad90da82 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -17,10 +17,6 @@ from bookwyrm.settings import PAGE_LENGTH from bookwyrm.views.helpers import is_api_request, get_user_from_username from bookwyrm.book_search import search -import logging -logger = logging.getLogger(__name__) - - # pylint: disable=no-self-use class Shelf(View): """shelf page""" diff --git a/docker-compose.yml b/docker-compose.yml index bb0acbdd1..4d4037681 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,8 +21,6 @@ services: - pgdata:/var/lib/postgresql/data networks: - main - ports: - - "5432:5432" web: build: . env_file: .env From aac8aa1adfd668e95595600d5b059b1b89655507 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Wed, 6 Dec 2023 11:36:15 -0800 Subject: [PATCH 07/56] Fixes formatting --- bookwyrm/book_search.py | 19 ++++++++++++++----- bookwyrm/views/shelf/shelf.py | 18 ++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index b3e66cdd6..669b7d377 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -14,7 +14,6 @@ from bookwyrm import connectors from bookwyrm.settings import MEDIA_FULL_URL - @overload def search( query: str, @@ -44,7 +43,7 @@ def search( min_confidence: float = 0, filters: Optional[list[Any]] = None, return_first: bool = False, - books: Optional[QuerySet[models.Edition]] = None + books: Optional[QuerySet[models.Edition]] = None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: """search your local database""" filters = filters or [] @@ -56,7 +55,9 @@ def search( # first, try searching unique identifiers # unique identifiers never have spaces, title/author usually do if not " " in query: - results = search_identifiers(query, *filters, return_first=return_first, books=books) + results = search_identifiers( + query, *filters, return_first=return_first, books=books + ) # if there were no identifier results... if not results: @@ -66,6 +67,7 @@ def search( ) return results + def isbn_search(query): """search your local database""" if not query: @@ -99,10 +101,17 @@ def format_search_result(search_result): def search_identifiers( - query, *filters, return_first=False, books=None, + query, + *filters, + return_first=False, + books=None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: + """search Editions by deduplication fields + + Best for cases when we can assume someone is searching for an exact match on + commonly unique data identifiers like isbn or specific library ids. + """ books = books or models.Edition.objects - """tries remote_id, isbn; defined as dedupe fields on the model""" if connectors.maybe_isbn(query): # Oh did you think the 'S' in ISBN stood for 'standard'? normalized_isbn = query.strip().upper().rjust(10, "0") diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index aad90da82..0617fcc56 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -1,7 +1,7 @@ """ shelf views """ from collections import namedtuple -from django.db.models import OuterRef, Subquery, F, Max, QuerySet +from django.db.models import OuterRef, Subquery, F, Max from django.contrib.auth.decorators import login_required from django.core.paginator import Paginator from django.http import HttpResponseBadRequest @@ -17,10 +17,12 @@ from bookwyrm.settings import PAGE_LENGTH from bookwyrm.views.helpers import is_api_request, get_user_from_username from bookwyrm.book_search import search + # pylint: disable=no-self-use class Shelf(View): """shelf page""" + # pylint: disable=R0914 def get(self, request, username, shelf_identifier=None): """display a shelf""" user = get_user_from_username(request.user, username) @@ -45,10 +47,14 @@ class Shelf(View): "Shelf", ("identifier", "name", "user", "books", "privacy") ) - books = models.Edition.viewer_aware_objects(request.user).filter( - # privacy is ensured because the shelves are already filtered above - shelfbook__shelf__in=shelves - ).distinct() + books = ( + models.Edition.viewer_aware_objects(request.user) + .filter( + # privacy is ensured because the shelves are already filtered above + shelfbook__shelf__in=shelves + ) + .distinct() + ) shelf = FakeShelf("all", _("All books"), user, books, "public") @@ -106,7 +112,7 @@ class Shelf(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), - "shelves_search_query": shelves_search_query + "shelves_search_query": shelves_search_query, } return TemplateResponse(request, "shelf/shelf.html", data) From 4a4046a704d9d2abfa72b51f1da33e6f6d0c908a Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Thu, 14 Dec 2023 11:30:01 -0800 Subject: [PATCH 08/56] Shows message if empty and renames "search" to "filter" --- bookwyrm/templates/shelf/search_filter_field.html | 9 --------- bookwyrm/templates/shelf/shelf.html | 6 ++++-- bookwyrm/templates/shelf/shelves_filter_field.html | 9 +++++++++ .../{search_filters.html => shelves_filters.html} | 2 +- bookwyrm/views/shelf/shelf.py | 11 +++++++---- 5 files changed, 21 insertions(+), 16 deletions(-) delete mode 100644 bookwyrm/templates/shelf/search_filter_field.html create mode 100644 bookwyrm/templates/shelf/shelves_filter_field.html rename bookwyrm/templates/shelf/{search_filters.html => shelves_filters.html} (66%) diff --git a/bookwyrm/templates/shelf/search_filter_field.html b/bookwyrm/templates/shelf/search_filter_field.html deleted file mode 100644 index 5641bae85..000000000 --- a/bookwyrm/templates/shelf/search_filter_field.html +++ /dev/null @@ -1,9 +0,0 @@ -{% extends 'snippets/filters_panel/filter_field.html' %} -{% load i18n %} - -{% block filter %} -
- - -
-{% endblock %} diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index 91d9ee99f..60b822322 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -101,18 +101,20 @@ {% plural %} {{ formatted_count }} books {% endblocktrans %} - {% if books.has_other_pages %} {% blocktrans trimmed with start=books.start_index end=books.end_index %} (showing {{ start }}-{{ end }}) {% endblocktrans %} {% endif %} + {% if shelves_filter_msg %} + - {{ shelves_filter_msg }} "{{ shelves_filter_query }}" + {% endif %} {% endif %} {% endwith %} {% if books|length > 0 %} - {% include 'shelf/search_filters.html' with user=user query=query %} + {% include 'shelf/shelves_filters.html' with user=user query=query %} {% endif %}
diff --git a/bookwyrm/templates/shelf/shelves_filter_field.html b/bookwyrm/templates/shelf/shelves_filter_field.html new file mode 100644 index 000000000..707f033ea --- /dev/null +++ b/bookwyrm/templates/shelf/shelves_filter_field.html @@ -0,0 +1,9 @@ +{% extends 'snippets/filters_panel/filter_field.html' %} +{% load i18n %} + +{% block filter %} +
+ + +
+{% endblock %} diff --git a/bookwyrm/templates/shelf/search_filters.html b/bookwyrm/templates/shelf/shelves_filters.html similarity index 66% rename from bookwyrm/templates/shelf/search_filters.html rename to bookwyrm/templates/shelf/shelves_filters.html index 840eec57e..ad7fc3dbc 100644 --- a/bookwyrm/templates/shelf/search_filters.html +++ b/bookwyrm/templates/shelf/shelves_filters.html @@ -1,5 +1,5 @@ {% extends 'snippets/filters_panel/filters_panel.html' %} {% block filter_fields %} - {% include 'shelf/search_filter_field.html' %} + {% include 'shelf/shelves_filter_field.html' %} {% endblock %} diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 0617fcc56..2b9ab176a 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -34,7 +34,8 @@ class Shelf(View): else: shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all() - shelves_search_query = request.GET.get("shelves_q") + shelves_filter_query = request.GET.get("filter") + shelves_filter_msg = "" # get the shelf and make sure the logged in user should be able to see it if shelf_identifier: @@ -92,8 +93,9 @@ class Shelf(View): books = sort_books(books, request.GET.get("sort")) - if shelves_search_query: - books = search(shelves_search_query, books=books) + if shelves_filter_query: + books = search(shelves_filter_query, books=books) or books + shelves_filter_msg = "We couldn't find any books that matched" paginated = Paginator( books, @@ -112,7 +114,8 @@ class Shelf(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), - "shelves_search_query": shelves_search_query, + "shelves_filter_query": shelves_filter_query, + "shelves_filter_msg": shelves_filter_msg, } return TemplateResponse(request, "shelf/shelf.html", data) From bd3acdbf31887801138324574726cff2e6c21a6f Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Thu, 14 Dec 2023 12:33:27 -0800 Subject: [PATCH 09/56] Puts string in template --- bookwyrm/templates/shelf/shelf.html | 4 ++-- bookwyrm/views/shelf/shelf.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index 60b822322..d224b0a4f 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -106,8 +106,8 @@ (showing {{ start }}-{{ end }}) {% endblocktrans %} {% endif %} - {% if shelves_filter_msg %} - - {{ shelves_filter_msg }} "{{ shelves_filter_query }}" + {% if show_shelves_filter_msg %} + - {% trans "We couldn't find any books that matched" %} "{{ shelves_filter_query }}" {% endif %} {% endif %} diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 2b9ab176a..eae57b409 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -35,7 +35,7 @@ class Shelf(View): shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all() shelves_filter_query = request.GET.get("filter") - shelves_filter_msg = "" + show_shelves_filter_msg = False # get the shelf and make sure the logged in user should be able to see it if shelf_identifier: @@ -95,7 +95,7 @@ class Shelf(View): if shelves_filter_query: books = search(shelves_filter_query, books=books) or books - shelves_filter_msg = "We couldn't find any books that matched" + show_shelves_filter_msg = True paginated = Paginator( books, @@ -115,7 +115,7 @@ class Shelf(View): page.number, on_each_side=2, on_ends=1 ), "shelves_filter_query": shelves_filter_query, - "shelves_filter_msg": shelves_filter_msg, + "show_shelves_filter_msg": show_shelves_filter_msg, } return TemplateResponse(request, "shelf/shelf.html", data) From 44d21d1ba474ae71b05fd0195f9a7ae7c73719cc Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Thu, 14 Dec 2023 13:04:45 -0800 Subject: [PATCH 10/56] Updates view logic --- bookwyrm/templates/shelf/shelf.html | 14 +++++++------- bookwyrm/views/shelf/shelf.py | 4 +--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index d224b0a4f..faadd84ab 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -106,16 +106,11 @@ (showing {{ start }}-{{ end }}) {% endblocktrans %} {% endif %} - {% if show_shelves_filter_msg %} - - {% trans "We couldn't find any books that matched" %} "{{ shelves_filter_query }}" - {% endif %} {% endif %} {% endwith %} - {% if books|length > 0 %} - {% include 'shelf/shelves_filters.html' with user=user query=query %} - {% endif %} + {% include 'shelf/shelves_filters.html' with user=user query=query %} {% if is_self and shelf.id %} @@ -215,7 +210,12 @@ {% else %} -

{% trans "This shelf is empty." %}

+

{% if shelves_filter_query %} + {% trans "We couldn't find any books that matched" %} "{{ shelves_filter_query }}" + {% else %} + {% trans "This shelf is empty." %} + {% endif %} +

{% endif %} diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index eae57b409..17e17433f 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -94,8 +94,7 @@ class Shelf(View): books = sort_books(books, request.GET.get("sort")) if shelves_filter_query: - books = search(shelves_filter_query, books=books) or books - show_shelves_filter_msg = True + books = search(shelves_filter_query, books=books) paginated = Paginator( books, @@ -115,7 +114,6 @@ class Shelf(View): page.number, on_each_side=2, on_ends=1 ), "shelves_filter_query": shelves_filter_query, - "show_shelves_filter_msg": show_shelves_filter_msg, } return TemplateResponse(request, "shelf/shelf.html", data) From fb369584442c64e70e3ec41c2f39aed240741666 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Thu, 14 Dec 2023 13:47:51 -0800 Subject: [PATCH 11/56] Removes unused variable --- bookwyrm/views/shelf/shelf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 17e17433f..aac8085d0 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -35,7 +35,6 @@ class Shelf(View): shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all() shelves_filter_query = request.GET.get("filter") - show_shelves_filter_msg = False # get the shelf and make sure the logged in user should be able to see it if shelf_identifier: From a4172214d16f2515b72c238df47f15229b9578a4 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Fri, 15 Dec 2023 13:17:23 -0800 Subject: [PATCH 12/56] Updates size of filters panel label --- bookwyrm/views/shelf/shelf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index aac8085d0..a39512fe6 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -113,6 +113,7 @@ class Shelf(View): page.number, on_each_side=2, on_ends=1 ), "shelves_filter_query": shelves_filter_query, + "size": "small", } return TemplateResponse(request, "shelf/shelf.html", data) From b728bb43232651d968be7aa477c2c62570d4b427 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Sat, 16 Dec 2023 12:05:35 -0800 Subject: [PATCH 13/56] Uses block trans --- bookwyrm/templates/shelf/shelf.html | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index faadd84ab..71f4bc088 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -210,12 +210,17 @@ {% else %} -

{% if shelves_filter_query %} - {% trans "We couldn't find any books that matched" %} "{{ shelves_filter_query }}" - {% else %} - {% trans "This shelf is empty." %} - {% endif %} -

+

+ + {% if shelves_filter_query %} + {% blocktrans trimmed %} + We couldn't find any books that matched {{ shelves_filter_query }} + {% endblocktrans %} + {% else %} + {% trans "This shelf is empty." %} + {% endif %} + +

{% endif %} From 2a85378456ef26d50702fc35417928a2c99bd7d3 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 09:57:41 -0800 Subject: [PATCH 14/56] Removes part of migration causing upgrade issues --- bookwyrm/migrations/0184_auto_20231106_0421.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/bookwyrm/migrations/0184_auto_20231106_0421.py b/bookwyrm/migrations/0184_auto_20231106_0421.py index e8197dea1..23bacc502 100644 --- a/bookwyrm/migrations/0184_auto_20231106_0421.py +++ b/bookwyrm/migrations/0184_auto_20231106_0421.py @@ -22,17 +22,6 @@ def update_deleted_users(apps, schema_editor): ).update(is_deleted=True) -def erase_deleted_user_data(apps, schema_editor): - """Retroactively clear user data""" - for user in User.objects.filter(is_deleted=True): - user.erase_user_data() - user.save( - broadcast=False, - update_fields=["email", "avatar", "preview_image", "summary", "name"], - ) - user.erase_user_statuses(broadcast=False) - - class Migration(migrations.Migration): dependencies = [ @@ -43,7 +32,4 @@ class Migration(migrations.Migration): migrations.RunPython( update_deleted_users, reverse_code=migrations.RunPython.noop ), - migrations.RunPython( - erase_deleted_user_data, reverse_code=migrations.RunPython.noop - ), ] From 381490e31d80670e8bb790a97b46b30611e32b9e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 10:35:30 -0800 Subject: [PATCH 15/56] Adds management command to clear all deleted user data --- .../commands/erase_deleted_user_data.py | 40 +++++++++++++++++++ bookwyrm/models/user.py | 14 +++++++ bw-dev | 5 ++- 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 bookwyrm/management/commands/erase_deleted_user_data.py diff --git a/bookwyrm/management/commands/erase_deleted_user_data.py b/bookwyrm/management/commands/erase_deleted_user_data.py new file mode 100644 index 000000000..fd3c790ce --- /dev/null +++ b/bookwyrm/management/commands/erase_deleted_user_data.py @@ -0,0 +1,40 @@ +""" Erase any data stored about deleted users """ +import sys +from django.core.management.base import BaseCommand, CommandError +from bookwyrm import models +from bookwyrm.models.user import erase_user_data + +# pylint: disable=missing-function-docstring +class Command(BaseCommand): + """command-line options""" + + help = "Remove Two Factor Authorisation from user" + + def add_arguments(self, parser): # pylint: disable=no-self-use + parser.add_argument( + "--dryrun", + action="store_true", + help="Preview users to be cleared without altering the database", + ) + + def handle(self, *args, **options): # pylint: disable=unused-argument + + # Check for anything fishy + bad_state = models.User.objects.filter(is_deleted=True, is_active=True) + if bad_state.exists(): + raise CommandError( + f"{bad_state.count()} user(s) marked as both active and deleted" + ) + + deleted_users = models.User.objects.filter(is_deleted=True) + self.stdout.write(f"Found {deleted_users.count()} deleted users") + if options["dryrun"]: + self.stdout.write("\n".join(u.username for u in deleted_users[:5])) + if deleted_users.count() > 5: + self.stdout.write("... and more") + sys.exit() + + self.stdout.write("Erasing user data:") + for user_id in deleted_users.values_list("id", flat=True): + erase_user_data.delay(user_id) + self.stdout.write(".", ending="") diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 75ca1d527..89fd39b73 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -523,6 +523,20 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): return super().save(*args, **kwargs) +@app.task(queue=MISC) +def erase_user_data(user_id): + """Erase any custom data about this user asynchronously + This is for deleted historical user data that pre-dates data + being cleared automatically""" + user = User.objects.get(id=user_id) + user.erase_user_data() + user.save( + broadcast=False, + update_fields=["email", "avatar", "preview_image", "summary", "name"], + ) + user.erase_user_statuses(broadcast=False) + + @app.task(queue=MISC) def set_remote_server(user_id, allow_external_connections=False): """figure out the user's remote server in the background""" diff --git a/bw-dev b/bw-dev index 27c20fe45..5a36f78e0 100755 --- a/bw-dev +++ b/bw-dev @@ -246,6 +246,9 @@ case "$CMD" in remove_remote_user_preview_images) runweb python manage.py remove_remote_user_preview_images ;; + erase_deleted_user_data) + runweb python manage.py erase_deleted_user_data "$@" + ;; copy_media_to_s3) awscommand "bookwyrm_media_volume:/images"\ "s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\ @@ -297,7 +300,7 @@ case "$CMD" in echo "Unrecognised command. Try:" echo " setup" echo " up [container]" - echo " down" + echo " down" echo " service_ports_web" echo " initdb" echo " resetdb" From 01db77a74584c6df18bc8dc624cacf8dbebb30f6 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 18:29:55 -0800 Subject: [PATCH 16/56] Adds success message --- bookwyrm/management/commands/erase_deleted_user_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bookwyrm/management/commands/erase_deleted_user_data.py b/bookwyrm/management/commands/erase_deleted_user_data.py index fd3c790ce..40c3f042b 100644 --- a/bookwyrm/management/commands/erase_deleted_user_data.py +++ b/bookwyrm/management/commands/erase_deleted_user_data.py @@ -38,3 +38,6 @@ class Command(BaseCommand): for user_id in deleted_users.values_list("id", flat=True): erase_user_data.delay(user_id) self.stdout.write(".", ending="") + + self.stdout.write("") + self.stdout.write("Tasks created successfully") From db8c686dd35df0126ee770455651d99d15458c82 Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Wed, 3 Jan 2024 15:43:15 +0100 Subject: [PATCH 17/56] Include book Readtrhough in the csv export --- bookwyrm/views/preferences/export.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index f54d97ccb..c691e2f50 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -54,6 +54,7 @@ class Export(View): fields = ( ["title", "author_text"] + deduplication_fields + + ["start_date", "finish_date", "stopped_date"] + ["rating", "review_name", "review_cw", "review_content"] ) writer.writerow(fields) @@ -70,6 +71,18 @@ class Export(View): book.rating = review_rating.rating if review_rating else None + readthrough = ( + models.ReadThrough.objects.filter( + user=request.user, book=book + ) + .order_by("-finish_date") + .first() + ) + if readthrough: + book.start_date = readthrough.start_date + book.finish_date = readthrough.finish_date + book.stopped_date = readthrough.stopped_date + review = ( models.Review.objects.filter( user=request.user, book=book, content__isnull=False From 766a2163ddc745afe0ecc4871f38d37235cb2cb9 Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Wed, 3 Jan 2024 20:41:31 +0100 Subject: [PATCH 18/56] Code formatting --- bookwyrm/views/preferences/export.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index c691e2f50..5a3b0fd8e 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -72,9 +72,7 @@ class Export(View): book.rating = review_rating.rating if review_rating else None readthrough = ( - models.ReadThrough.objects.filter( - user=request.user, book=book - ) + models.ReadThrough.objects.filter(user=request.user, book=book) .order_by("-finish_date") .first() ) From ae5950f1871f2ac168c810a0a6bc98c7b5e37474 Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Thu, 4 Jan 2024 11:03:07 +0100 Subject: [PATCH 19/56] Add readthrough fields to text_export.py --- bookwyrm/tests/views/preferences/test_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/tests/views/preferences/test_export.py b/bookwyrm/tests/views/preferences/test_export.py index 4f498f589..d633ae952 100644 --- a/bookwyrm/tests/views/preferences/test_export.py +++ b/bookwyrm/tests/views/preferences/test_export.py @@ -66,7 +66,7 @@ class ExportViews(TestCase): # pylint: disable=line-too-long self.assertEqual( export.content, - b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,rating,review_name,review_cw,review_content\r\nTest Book,," + b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content\r\nTest Book,," + self.book.remote_id.encode("utf-8") - + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,\r\n", + + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,\r\n", ) From 9acb5f66fee9e64f43c703f1dd631146f35a7724 Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Thu, 4 Jan 2024 11:26:44 +0100 Subject: [PATCH 20/56] Convert DateTime to date --- bookwyrm/views/preferences/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 5a3b0fd8e..4353cf259 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -77,9 +77,9 @@ class Export(View): .first() ) if readthrough: - book.start_date = readthrough.start_date - book.finish_date = readthrough.finish_date - book.stopped_date = readthrough.stopped_date + book.start_date = readthrough.start_date.date() if readthrough.start_date else "" + book.finish_date = readthrough.finish_date.date() if readthrough.finish_date else "" + book.stopped_date = readthrough.stopped_date.date() if readthrough.stopped_date else "" review = ( models.Review.objects.filter( From 51cb70d34423afb65653947bff15a90a8fc3278e Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Thu, 4 Jan 2024 11:27:17 +0100 Subject: [PATCH 21/56] Change readhtrough order --- bookwyrm/views/preferences/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 4353cf259..c8badbc8c 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -73,7 +73,7 @@ class Export(View): readthrough = ( models.ReadThrough.objects.filter(user=request.user, book=book) - .order_by("-finish_date") + .order_by("-start_date","-finish_date") .first() ) if readthrough: From 30c9ec9611530ee8d98eaef8fc046c5e514d9f28 Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Thu, 4 Jan 2024 11:28:17 +0100 Subject: [PATCH 22/56] Prevent lint error See @hughrun 's explanation https://github.com/bookwyrm-social/bookwyrm/pull/3189#issuecomment-1876145423 --- bookwyrm/tests/views/preferences/test_export.py | 4 +++- bookwyrm/views/preferences/export.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bookwyrm/tests/views/preferences/test_export.py b/bookwyrm/tests/views/preferences/test_export.py index d633ae952..2ef0f96fb 100644 --- a/bookwyrm/tests/views/preferences/test_export.py +++ b/bookwyrm/tests/views/preferences/test_export.py @@ -38,6 +38,8 @@ class ExportViews(TestCase): parent_work=self.work, isbn_13="9781234567890", bnf_id="beep", + start_date="2023-01-04", + finish_date="2024-01-04" ) def setUp(self): @@ -68,5 +70,5 @@ class ExportViews(TestCase): export.content, b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content\r\nTest Book,," + self.book.remote_id.encode("utf-8") - + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,\r\n", + + b",,,,,beep,,,,,,123456789X,9781234567890,,,2023-01-04,2024-01-04,,,,\r\n", ) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index c8badbc8c..2450a427d 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -103,7 +103,7 @@ class Export(View): ) -# pylint: disable=no-self-use +# pylint: disable=no-self-use,too-many-locals @method_decorator(login_required, name="dispatch") class ExportUser(View): """Let users export user data to import into another Bookwyrm instance""" From ebcc81dd73b295141a16407fded008d3c0abb85d Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Thu, 4 Jan 2024 11:33:26 +0100 Subject: [PATCH 23/56] Revert changes to default book These changes were introduced by mistake in my previous commit. --- bookwyrm/tests/views/preferences/test_export.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bookwyrm/tests/views/preferences/test_export.py b/bookwyrm/tests/views/preferences/test_export.py index 2ef0f96fb..74201ddad 100644 --- a/bookwyrm/tests/views/preferences/test_export.py +++ b/bookwyrm/tests/views/preferences/test_export.py @@ -37,9 +37,7 @@ class ExportViews(TestCase): remote_id="https://example.com/book/1", parent_work=self.work, isbn_13="9781234567890", - bnf_id="beep", - start_date="2023-01-04", - finish_date="2024-01-04" + bnf_id="beep" ) def setUp(self): @@ -70,5 +68,5 @@ class ExportViews(TestCase): export.content, b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content\r\nTest Book,," + self.book.remote_id.encode("utf-8") - + b",,,,,beep,,,,,,123456789X,9781234567890,,,2023-01-04,2024-01-04,,,,\r\n", + + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,\r\n", ) From c2622a510cd40eb3883eae4d01ccf2095754aa3b Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Thu, 4 Jan 2024 11:40:40 +0100 Subject: [PATCH 24/56] Change else statement to None vs "" --- bookwyrm/views/preferences/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 2450a427d..a6d51e24a 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -77,9 +77,9 @@ class Export(View): .first() ) if readthrough: - book.start_date = readthrough.start_date.date() if readthrough.start_date else "" - book.finish_date = readthrough.finish_date.date() if readthrough.finish_date else "" - book.stopped_date = readthrough.stopped_date.date() if readthrough.stopped_date else "" + book.start_date = readthrough.start_date.date() if readthrough.start_date else None + book.finish_date = readthrough.finish_date.date() if readthrough.finish_date else None + book.stopped_date = readthrough.stopped_date.date() if readthrough.stopped_date else None review = ( models.Review.objects.filter( From f267fc32355542776828e7894fe1f87c5ca68b75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 17:42:04 +0000 Subject: [PATCH 25/56] Bump pycryptodome from 3.16.0 to 3.19.1 Bumps [pycryptodome](https://github.com/Legrandin/pycryptodome) from 3.16.0 to 3.19.1. - [Release notes](https://github.com/Legrandin/pycryptodome/releases) - [Changelog](https://github.com/Legrandin/pycryptodome/blob/master/Changelog.rst) - [Commits](https://github.com/Legrandin/pycryptodome/compare/v3.16.0...v3.19.1) --- updated-dependencies: - dependency-name: pycryptodome dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0b09f3c19..6509effc7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ libsass==0.22.0 Markdown==3.4.1 Pillow==10.0.1 psycopg2==2.9.5 -pycryptodome==3.16.0 +pycryptodome==3.19.1 python-dateutil==2.8.2 redis==4.5.4 requests==2.31.0 From 93cab480d6a7cc1d8897d8a0a765c6ce9f5301a4 Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Sat, 6 Jan 2024 09:50:14 +0100 Subject: [PATCH 26/56] Code format --- bookwyrm/tests/views/preferences/test_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/tests/views/preferences/test_export.py b/bookwyrm/tests/views/preferences/test_export.py index 74201ddad..d633ae952 100644 --- a/bookwyrm/tests/views/preferences/test_export.py +++ b/bookwyrm/tests/views/preferences/test_export.py @@ -37,7 +37,7 @@ class ExportViews(TestCase): remote_id="https://example.com/book/1", parent_work=self.work, isbn_13="9781234567890", - bnf_id="beep" + bnf_id="beep", ) def setUp(self): From ce18d343e8de7a40b2da17fae0ebff1c065eda9f Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Sat, 6 Jan 2024 09:53:22 +0100 Subject: [PATCH 27/56] Fix pylint error and code format --- bookwyrm/views/preferences/export.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index a6d51e24a..3443be461 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -17,7 +17,7 @@ from bookwyrm import models from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm.settings import PAGE_LENGTH -# pylint: disable=no-self-use +# pylint: disable=no-self-use,too-many-locals @method_decorator(login_required, name="dispatch") class Export(View): """Let users export data""" @@ -73,13 +73,21 @@ class Export(View): readthrough = ( models.ReadThrough.objects.filter(user=request.user, book=book) - .order_by("-start_date","-finish_date") + .order_by("-start_date", "-finish_date") .first() ) if readthrough: - book.start_date = readthrough.start_date.date() if readthrough.start_date else None - book.finish_date = readthrough.finish_date.date() if readthrough.finish_date else None - book.stopped_date = readthrough.stopped_date.date() if readthrough.stopped_date else None + book.start_date = ( + readthrough.start_date.date() if readthrough.start_date else None + ) + book.finish_date = ( + readthrough.finish_date.date() if readthrough.finish_date else None + ) + book.stopped_date = ( + readthrough.stopped_date.date() + if readthrough.stopped_date + else None + ) review = ( models.Review.objects.filter( @@ -103,7 +111,7 @@ class Export(View): ) -# pylint: disable=no-self-use,too-many-locals +# pylint: disable=no-self-use @method_decorator(login_required, name="dispatch") class ExportUser(View): """Let users export user data to import into another Bookwyrm instance""" From 83ff880603745adf2ef92b8ca489f9d2a7ea1d88 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 7 Jan 2024 08:31:48 -0800 Subject: [PATCH 28/56] Revert "Don't show notification for user follow request if the user is inactive" --- bookwyrm/templates/notifications/item.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bookwyrm/templates/notifications/item.html b/bookwyrm/templates/notifications/item.html index a1329d31e..ac60ad46f 100644 --- a/bookwyrm/templates/notifications/item.html +++ b/bookwyrm/templates/notifications/item.html @@ -10,9 +10,7 @@ {% elif notification.notification_type == 'FOLLOW' %} {% include 'notifications/items/follow.html' %} {% elif notification.notification_type == 'FOLLOW_REQUEST' %} - {% if notification.related_users.0.is_active %} - {% include 'notifications/items/follow_request.html' %} - {% endif %} + {% include 'notifications/items/follow_request.html' %} {% elif notification.notification_type == 'IMPORT' %} {% include 'notifications/items/import.html' %} {% elif notification.notification_type == 'USER_IMPORT' %} From 1a682753c0f126a8319389397d7558d88183b989 Mon Sep 17 00:00:00 2001 From: Rohan Sureshkumar Date: Tue, 9 Jan 2024 15:31:05 +0530 Subject: [PATCH 29/56] Issue-3187: changes --- bookwyrm/templates/feed/feed.html | 2 +- bookwyrm/views/feed.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bookwyrm/templates/feed/feed.html b/bookwyrm/templates/feed/feed.html index 7ecf10b70..820314b7a 100644 --- a/bookwyrm/templates/feed/feed.html +++ b/bookwyrm/templates/feed/feed.html @@ -33,7 +33,7 @@ - {% if request.user.show_goal and not goal and tab.key == 'home' %} + {% if request.user.show_goal and not goal and tab.key == 'home' and has_read_throughs %} {% now 'Y' as year %}
{% include 'feed/goal_card.html' with year=year %} diff --git a/bookwyrm/views/feed.py b/bookwyrm/views/feed.py index 17218b93e..381d233e9 100644 --- a/bookwyrm/views/feed.py +++ b/bookwyrm/views/feed.py @@ -52,6 +52,8 @@ class Feed(View): suggestions = suggested_users.get_suggestions(request.user) + readthroughs = models.ReadThrough.objects.filter(user=request.user) + data = { **feed_page_data(request.user), **{ @@ -66,6 +68,7 @@ class Feed(View): "path": f"/{tab['key']}", "annual_summary_year": get_annual_summary_year(), "has_tour": True, + "has_read_throughs": True if len(readthroughs) else False, }, } return TemplateResponse(request, "feed/feed.html", data) From 854eb36618fe4371b37b4da3a86c68d6778d195b Mon Sep 17 00:00:00 2001 From: Carlos Camara Date: Sat, 13 Jan 2024 16:43:41 +0000 Subject: [PATCH 30/56] Export bookshelves and review date --- .../tests/views/preferences/test_export.py | 9 +++-- bookwyrm/views/preferences/export.py | 36 +++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/bookwyrm/tests/views/preferences/test_export.py b/bookwyrm/tests/views/preferences/test_export.py index d633ae952..7a1bcd6db 100644 --- a/bookwyrm/tests/views/preferences/test_export.py +++ b/bookwyrm/tests/views/preferences/test_export.py @@ -18,7 +18,9 @@ class ExportViews(TestCase): """viewing and creating statuses""" @classmethod - def setUpTestData(self): # pylint: disable=bad-classmethod-argument + def setUpTestData( + self, + ): # pylint: disable=bad-classmethod-argument, disable=invalid-name """we need basic test data and mocks""" with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" @@ -40,6 +42,7 @@ class ExportViews(TestCase): bnf_id="beep", ) + # pylint: disable=invalid-name def setUp(self): """individual test setup""" self.factory = RequestFactory() @@ -66,7 +69,7 @@ class ExportViews(TestCase): # pylint: disable=line-too-long self.assertEqual( export.content, - b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content\r\nTest Book,," + b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date\r\nTest Book,," + self.book.remote_id.encode("utf-8") - + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,\r\n", + + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,,,,,\r\n", ) diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index 3443be461..d16f3aaa3 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -17,6 +17,7 @@ from bookwyrm import models from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm.settings import PAGE_LENGTH + # pylint: disable=no-self-use,too-many-locals @method_decorator(login_required, name="dispatch") class Export(View): @@ -54,8 +55,19 @@ class Export(View): fields = ( ["title", "author_text"] + deduplication_fields - + ["start_date", "finish_date", "stopped_date"] - + ["rating", "review_name", "review_cw", "review_content"] + + [ + "start_date", + "finish_date", + "stopped_date", + "rating", + "review_name", + "review_cw", + "review_content", + "review_published", + "shelf", + "shelf_name", + "shelf_date", + ] ) writer.writerow(fields) @@ -97,9 +109,27 @@ class Export(View): .first() ) if review: + book.review_published = ( + review.published_date.date() if review.published_date else None + ) book.review_name = review.name book.review_cw = review.content_warning - book.review_content = review.raw_content + book.review_content = ( + review.raw_content if review.raw_content else review.content + ) # GoodReads imported reviews do not have raw_content, but content. + + shelfbook = ( + models.ShelfBook.objects.filter(user=request.user, book=book) + .order_by("-shelved_date", "-created_date", "-updated_date") + .last() + ) + if shelfbook: + book.shelf = shelfbook.shelf.identifier + book.shelf_name = shelfbook.shelf.name + book.shelf_date = ( + shelfbook.shelved_date.date() if shelfbook.shelved_date else None + ) + writer.writerow([getattr(book, field, "") or "" for field in fields]) return HttpResponse( From 9a487b0442c0d9e94b511491bf61bd00b3cfebfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adeodato=20Sim=C3=B3?= Date: Sat, 13 Jan 2024 17:51:14 +0100 Subject: [PATCH 31/56] Ensure dev-tools uses bookworm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 1937177e1 ("dev-tools: use apt source for Node instead of setup script"), I introduced the use of `Signed-By` with a public key block, which is only supported in bookworm (bullseye only supports fingerprints, TTBOMK). Python's Docker images already use bookworm by default, but we explicitly require it now to avoid build errors if someone has a very old image laying around (see, e.g., #3190). (This can be dropped after Debian 13 ‘trixie’ is released.) --- dev-tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/Dockerfile b/dev-tools/Dockerfile index 3b7740a78..6c132944f 100644 --- a/dev-tools/Dockerfile +++ b/dev-tools/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.9-bookworm WORKDIR /app/dev-tools ENV PATH="/app/dev-tools/node_modules/.bin:$PATH" From 5ef104b802dce740835f4a34bb4403459e2791bc Mon Sep 17 00:00:00 2001 From: Rohan Sureshkumar Date: Mon, 15 Jan 2024 17:22:33 +0530 Subject: [PATCH 32/56] Issue-3187: addressing review comments --- bookwyrm/templates/feed/feed.html | 4 ++-- bookwyrm/views/feed.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bookwyrm/templates/feed/feed.html b/bookwyrm/templates/feed/feed.html index 820314b7a..56c380202 100644 --- a/bookwyrm/templates/feed/feed.html +++ b/bookwyrm/templates/feed/feed.html @@ -33,7 +33,7 @@ - {% if request.user.show_goal and not goal and tab.key == 'home' and has_read_throughs %} + {% if request.user.show_goal and not goal and tab.key == 'home' %} {% now 'Y' as year %}
{% include 'feed/goal_card.html' with year=year %} @@ -41,7 +41,7 @@
{% endif %} - {% if annual_summary_year and tab.key == 'home' %} + {% if annual_summary_year and tab.key == 'home' and has_read_throughs %} {% endif %} - {% if annual_summary_year and tab.key == 'home' and has_read_throughs %} + {% if annual_summary_year and tab.key == 'home' and has_summary_read_throughs %}