From 482005f3049ba8a1c533983f5751b18e1f6ba3fe Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 5 Jun 2022 20:03:36 +0000 Subject: [PATCH 1/3] Update preview image generation to only query ids Previously we were querying the full book objects just to get a list of id's, which is much slower and also takes a lot more memory, which can cause the process to be killed on memory-limited machines with a large number of books. Instead, since we're just dispatching jobs here, we can just ask for the id's, which is faster and much more practical memory-wise. The map is a little annoying, I didn't see a way to directly get just a list of the value of one field, so we have to get a list of dictionairies with one key and then pull that key out. Whatevs. --- bookwyrm/management/commands/generate_preview_images.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bookwyrm/management/commands/generate_preview_images.py b/bookwyrm/management/commands/generate_preview_images.py index 0454e5e51..b00fafb10 100644 --- a/bookwyrm/management/commands/generate_preview_images.py +++ b/bookwyrm/management/commands/generate_preview_images.py @@ -56,12 +56,12 @@ class Command(BaseCommand): self.stdout.write(" OK 🖼") # Books - books = models.Book.objects.select_subclasses().filter() + book_ids = [values["id"] for values in models.Book.objects.select_subclasses().filter().values('id')] self.stdout.write( - " → Book preview images ({}): ".format(len(books)), ending="" + " → Book preview images ({}): ".format(len(book_ids)), ending="" ) - for book in books: - preview_images.generate_edition_preview_image_task.delay(book.id) + for book_id in book_ids: + preview_images.generate_edition_preview_image_task.delay(book_id) self.stdout.write(".", ending="") self.stdout.write(" OK 🖼") From b744ff7836dbf4e42b52ccd4d6ddb747270009d8 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 5 Jun 2022 13:40:01 -0700 Subject: [PATCH 2/3] Run black --- bookwyrm/management/commands/generate_preview_images.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bookwyrm/management/commands/generate_preview_images.py b/bookwyrm/management/commands/generate_preview_images.py index b00fafb10..7952803d8 100644 --- a/bookwyrm/management/commands/generate_preview_images.py +++ b/bookwyrm/management/commands/generate_preview_images.py @@ -56,7 +56,12 @@ class Command(BaseCommand): self.stdout.write(" OK 🖼") # Books - book_ids = [values["id"] for values in models.Book.objects.select_subclasses().filter().values('id')] + book_ids = [ + values["id"] + for values in models.Book.objects.select_subclasses() + .filter() + .values("id") + ] self.stdout.write( " → Book preview images ({}): ".format(len(book_ids)), ending="" ) From 7f5d47a36f765d54ad47c8b272c5b778776e8f75 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Tue, 7 Jun 2022 23:14:45 -0700 Subject: [PATCH 3/3] Use values_list with flat, yay! --- .../management/commands/generate_preview_images.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bookwyrm/management/commands/generate_preview_images.py b/bookwyrm/management/commands/generate_preview_images.py index 7952803d8..9ff16c26a 100644 --- a/bookwyrm/management/commands/generate_preview_images.py +++ b/bookwyrm/management/commands/generate_preview_images.py @@ -56,12 +56,12 @@ class Command(BaseCommand): self.stdout.write(" OK 🖼") # Books - book_ids = [ - values["id"] - for values in models.Book.objects.select_subclasses() + book_ids = ( + models.Book.objects.select_subclasses() .filter() - .values("id") - ] + .values_list("id", flat=True) + ) + self.stdout.write( " → Book preview images ({}): ".format(len(book_ids)), ending="" )