diff --git a/bookwyrm/migrations/0192_auto_20240114_0055.py b/bookwyrm/migrations/0192_auto_20240114_0055.py index f4d324f7f..824439728 100644 --- a/bookwyrm/migrations/0192_auto_20240114_0055.py +++ b/bookwyrm/migrations/0192_auto_20240114_0055.py @@ -9,45 +9,84 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('bookwyrm', '0191_merge_20240102_0326'), + ("bookwyrm", "0191_merge_20240102_0326"), ] operations = [ migrations.AddField( - model_name='bookwyrmexportjob', - name='export_json', - field=models.JSONField(encoder=django.core.serializers.json.DjangoJSONEncoder, null=True), + model_name="bookwyrmexportjob", + name="export_json", + field=models.JSONField( + encoder=django.core.serializers.json.DjangoJSONEncoder, null=True + ), ), migrations.AddField( - model_name='bookwyrmexportjob', - name='json_completed', + model_name="bookwyrmexportjob", + name="json_completed", field=models.BooleanField(default=False), ), migrations.AlterField( - model_name='bookwyrmexportjob', - name='export_data', - field=models.FileField(null=True, storage=bookwyrm.storage_backends.ExportsFileStorage, upload_to=''), + model_name="bookwyrmexportjob", + name="export_data", + field=models.FileField( + null=True, + storage=bookwyrm.storage_backends.ExportsFileStorage, + upload_to="", + ), ), migrations.CreateModel( - name='AddFileToTar', + name="AddFileToTar", fields=[ - ('childjob_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='bookwyrm.childjob')), - ('parent_export_job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='child_edition_export_jobs', to='bookwyrm.bookwyrmexportjob')), + ( + "childjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.childjob", + ), + ), + ( + "parent_export_job", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="child_edition_export_jobs", + to="bookwyrm.bookwyrmexportjob", + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, - bases=('bookwyrm.childjob',), + bases=("bookwyrm.childjob",), ), migrations.CreateModel( - name='AddBookToUserExportJob', + name="AddBookToUserExportJob", fields=[ - ('childjob_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='bookwyrm.childjob')), - ('edition', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='bookwyrm.edition')), + ( + "childjob_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="bookwyrm.childjob", + ), + ), + ( + "edition", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="bookwyrm.edition", + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, - bases=('bookwyrm.childjob',), + bases=("bookwyrm.childjob",), ), ] diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index 12a9792e2..2d1c0d94f 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -24,6 +24,7 @@ from bookwyrm.utils.tar import BookwyrmTarFile logger = logging.getLogger(__name__) + class BookwyrmExportJob(ParentJob): """entry for a specific request to export a bookwyrm user""" @@ -32,11 +33,12 @@ class BookwyrmExportJob(ParentJob): else: storage = storage_backends.ExportsFileStorage - export_data = FileField(null=True, storage=storage) # use custom storage backend here + export_data = FileField( + null=True, storage=storage + ) # use custom storage backend here export_json = JSONField(null=True, encoder=DjangoJSONEncoder) json_completed = BooleanField(default=False) - def start_job(self): """Start the job""" @@ -44,7 +46,6 @@ class BookwyrmExportJob(ParentJob): self.task_id = task.id self.save(update_fields=["task_id"]) - def notify_child_job_complete(self): """let the job know when the items get work done""" @@ -63,9 +64,8 @@ class BookwyrmExportJob(ParentJob): # add json file to tarfile tar_job = AddFileToTar.objects.create( - parent_job=self, - parent_export_job=self - ) + parent_job=self, parent_export_job=self + ) tar_job.start_job() except Exception as err: # pylint: disable=broad-except @@ -116,7 +116,9 @@ class AddBookToUserExportJob(ChildJob): # ListItems include "notes" and "approved" so we need them # even though we know it's this book book["lists"] = [] - list_items = ListItem.objects.filter(book=self.edition, user=self.parent_job.user).distinct() + list_items = ListItem.objects.filter( + book=self.edition, user=self.parent_job.user + ).distinct() for item in list_items: list_info = item.book_list.to_activity() @@ -133,16 +135,18 @@ class AddBookToUserExportJob(ChildJob): for status in ["comments", "quotations", "reviews"]: book[status] = [] - - comments = Comment.objects.filter(user=self.parent_job.user, book=self.edition).all() + comments = Comment.objects.filter( + user=self.parent_job.user, book=self.edition + ).all() for status in comments: obj = status.to_activity() obj["progress"] = status.progress obj["progress_mode"] = status.progress_mode book["comments"].append(obj) - - quotes = Quotation.objects.filter(user=self.parent_job.user, book=self.edition).all() + quotes = Quotation.objects.filter( + user=self.parent_job.user, book=self.edition + ).all() for status in quotes: obj = status.to_activity() obj["position"] = status.position @@ -150,15 +154,18 @@ class AddBookToUserExportJob(ChildJob): obj["position_mode"] = status.position_mode book["quotations"].append(obj) - - reviews = Review.objects.filter(user=self.parent_job.user, book=self.edition).all() + reviews = Review.objects.filter( + user=self.parent_job.user, book=self.edition + ).all() for status in reviews: obj = status.to_activity() book["reviews"].append(obj) # readthroughs can't be serialized to activity book_readthroughs = ( - ReadThrough.objects.filter(user=self.parent_job.user, book=self.edition).distinct().values() + ReadThrough.objects.filter(user=self.parent_job.user, book=self.edition) + .distinct() + .values() ) book["readthroughs"] = list(book_readthroughs) @@ -167,7 +174,9 @@ class AddBookToUserExportJob(ChildJob): self.complete_job() except Exception as err: # pylint: disable=broad-except - logger.exception("AddBookToUserExportJob %s Failed with error: %s", self.id, err) + logger.exception( + "AddBookToUserExportJob %s Failed with error: %s", self.id, err + ) self.set_status("failed") @@ -176,8 +185,7 @@ class AddFileToTar(ChildJob): parent_export_job = ForeignKey( BookwyrmExportJob, on_delete=CASCADE, related_name="child_edition_export_jobs" - ) # TODO: do we actually need this? Does self.parent_job.export_data work? - + ) # TODO: do we actually need this? Does self.parent_job.export_data work? def start_job(self): """Start the job""" @@ -188,7 +196,7 @@ class AddFileToTar(ChildJob): # but Hugh couldn't make that work try: - task_id=self.parent_export_job.task_id + task_id = self.parent_export_job.task_id export_data = self.parent_export_job.export_data export_json = self.parent_export_job.export_json json_data = DjangoJSONEncoder().encode(export_json) @@ -198,27 +206,19 @@ class AddFileToTar(ChildJob): if settings.USE_S3: s3_job = S3Tar( settings.AWS_STORAGE_BUCKET_NAME, - f"exports/{str(self.parent_export_job.task_id)}.tar.gz" + f"exports/{str(self.parent_export_job.task_id)}.tar.gz", ) # TODO: either encrypt the file or we will need to get it to the user # from this secure part of the bucket export_data.save("archive.json", ContentFile(json_data.encode("utf-8"))) - s3_job.add_file( - f"exports/{export_data.name}" - ) - s3_job.add_file( - f"images/{user.avatar.name}", - folder="avatar" - ) + s3_job.add_file(f"exports/{export_data.name}") + s3_job.add_file(f"images/{user.avatar.name}", folder="avatar") for book in editions: if getattr(book, "cover", False): cover_name = f"images/{book.cover.name}" - s3_job.add_file( - cover_name, - folder="covers" - ) + s3_job.add_file(cover_name, folder="covers") s3_job.tar() # delete export json as soon as it's tarred @@ -228,7 +228,7 @@ class AddFileToTar(ChildJob): else: # TODO: is the export_data file open to the world? - logger.info( "export file URL: %s",export_data.url) + logger.info("export file URL: %s", export_data.url) export_data.open("wb") with BookwyrmTarFile.open(mode="w:gz", fileobj=export_data) as tar: @@ -237,7 +237,9 @@ class AddFileToTar(ChildJob): # Add avatar image if present if getattr(user, "avatar", False): - tar.add_image(user.avatar, filename="avatar", directory=f"avatar/") # TODO: does this work? + tar.add_image( + user.avatar, filename="avatar", directory=f"avatar/" + ) # TODO: does this work? for book in editions: if getattr(book, "cover", False): @@ -245,7 +247,6 @@ class AddFileToTar(ChildJob): export_data.close() - self.complete_job() except Exception as err: # pylint: disable=broad-except @@ -277,6 +278,7 @@ def start_export_task(**kwargs): logger.exception("User Export Job %s Failed with error: %s", job.id, err) job.set_status("failed") + @app.task(queue=IMPORTS, base=ParentTask) def export_saved_lists_task(**kwargs): """add user saved lists to export JSON""" @@ -381,16 +383,23 @@ def trigger_books_jobs(**kwargs): for edition in editions: try: - edition_job = AddBookToUserExportJob.objects.create(edition=edition, parent_job=job) + edition_job = AddBookToUserExportJob.objects.create( + edition=edition, parent_job=job + ) edition_job.start_job() except Exception as err: # pylint: disable=broad-except - logger.exception("AddBookToUserExportJob %s Failed with error: %s", edition_job.id, err) + logger.exception( + "AddBookToUserExportJob %s Failed with error: %s", + edition_job.id, + err, + ) edition_job.set_status("failed") except Exception as err: # pylint: disable=broad-except logger.exception("trigger_books_jobs %s Failed with error: %s", job.id, err) job.set_status("failed") + def get_books_for_user(user): """Get all the books and editions related to a user""" diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 7896850e3..7c8947521 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -442,4 +442,6 @@ if HTTP_X_FORWARDED_PROTO: # Do not change this setting unless you already have an existing # user with the same username - in which case you should change it! INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" -DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 20) # 20MB TEMPORARY FIX WHILST WORKING ON THIS \ No newline at end of file +DATA_UPLOAD_MAX_MEMORY_SIZE = ( + 1024**2 * 20 +) # 20MB TEMPORARY FIX WHILST WORKING ON THIS diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py index c97b4e848..87c29ae70 100644 --- a/bookwyrm/storage_backends.py +++ b/bookwyrm/storage_backends.py @@ -63,15 +63,17 @@ class AzureImagesStorage(AzureStorage): # pylint: disable=abstract-method location = "images" overwrite_files = False + class ExportsFileStorage(FileSystemStorage): # pylint: disable=abstract-method """Storage class for exports contents with local files""" location = "exports" overwrite_files = False + class ExportsS3Storage(S3Boto3Storage): # pylint: disable=abstract-method """Storage class for exports contents with S3""" location = "exports" default_acl = None - overwrite_files = False \ No newline at end of file + overwrite_files = False diff --git a/exports/6ee95f7f-58cd-4bff-9d41-1ac2b3db6187 b/exports/6ee95f7f-58cd-4bff-9d41-1ac2b3db6187 deleted file mode 100644 index d7166b703..000000000 Binary files a/exports/6ee95f7f-58cd-4bff-9d41-1ac2b3db6187 and /dev/null differ diff --git a/exports/ba15a57f-e29e-4a29-aaf4-306b66960273 b/exports/ba15a57f-e29e-4a29-aaf4-306b66960273 deleted file mode 100644 index 318069303..000000000 Binary files a/exports/ba15a57f-e29e-4a29-aaf4-306b66960273 and /dev/null differ