diff --git a/bookwyrm/migrations/0183_auto_20231021_2050.py b/bookwyrm/migrations/0183_auto_20231021_2050.py new file mode 100644 index 000000000..c960fe5bd --- /dev/null +++ b/bookwyrm/migrations/0183_auto_20231021_2050.py @@ -0,0 +1,85 @@ +# Generated by Django 3.2.20 on 2023-10-21 20:50 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0182_merge_20230905_2240"), + ] + + operations = [ + migrations.AddField( + model_name="notification", + name="related_user_export", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="bookwyrm.bookwyrmexportjob", + ), + ), + migrations.AlterField( + model_name="childjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + migrations.AlterField( + model_name="notification", + name="notification_type", + field=models.CharField( + choices=[ + ("FAVORITE", "Favorite"), + ("REPLY", "Reply"), + ("MENTION", "Mention"), + ("TAG", "Tag"), + ("FOLLOW", "Follow"), + ("FOLLOW_REQUEST", "Follow Request"), + ("BOOST", "Boost"), + ("IMPORT", "Import"), + ("USER_IMPORT", "User Import"), + ("USER_EXPORT", "User Export"), + ("ADD", "Add"), + ("REPORT", "Report"), + ("LINK_DOMAIN", "Link Domain"), + ("INVITE", "Invite"), + ("ACCEPT", "Accept"), + ("JOIN", "Join"), + ("LEAVE", "Leave"), + ("REMOVE", "Remove"), + ("GROUP_PRIVACY", "Group Privacy"), + ("GROUP_NAME", "Group Name"), + ("GROUP_DESCRIPTION", "Group Description"), + ], + max_length=255, + ), + ), + migrations.AlterField( + model_name="parentjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ("failed", "Failed"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + ] diff --git a/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py b/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py new file mode 100644 index 000000000..24b4dad37 --- /dev/null +++ b/bookwyrm/migrations/0184_sitesettings_user_import_time_limit.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-10-22 02:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0183_auto_20231021_2050"), + ] + + operations = [ + migrations.AddField( + model_name="sitesettings", + name="user_import_time_limit", + field=models.IntegerField(default=48), + ), + ] diff --git a/bookwyrm/models/bookwyrm_export_job.py b/bookwyrm/models/bookwyrm_export_job.py index c262d9b5c..65f209905 100644 --- a/bookwyrm/models/bookwyrm_export_job.py +++ b/bookwyrm/models/bookwyrm_export_job.py @@ -35,13 +35,17 @@ def start_export_task(**kwargs): # don't start the job if it was stopped from the UI if job.complete: return - - # This is where ChildJobs get made - job.export_data = ContentFile(b"", str(uuid4())) - - json_data = json_export(job.user) - tar_export(json_data, job.user, job.export_data) - + try: + # This is where ChildJobs get made + job.export_data = ContentFile(b"", str(uuid4())) + json_data = json_export(job.user) + tar_export(json_data, job.user, job.export_data) + except Exception as err: # pylint: disable=broad-except + logger.exception("User Export Job %s Failed with error: %s", job.id, err) + job.set_status("failed") + job.set_status( + "complete" + ) # need to explicitly set this here to trigger notifications job.save(update_fields=["export_data"]) @@ -56,7 +60,8 @@ def tar_export(json_data: str, user, f): editions, books = get_books_for_user(user) for book in editions: - tar.add_image(book.cover) + if getattr(book, "cover", False): + tar.add_image(book.cover) f.close() @@ -153,20 +158,12 @@ def json_export(user): comments = models.Comment.objects.filter(user=user, book=book["id"]).distinct() book["comments"] = list(comments.values()) - logger.error("FINAL COMMENTS") - logger.error(book["comments"]) # quotes quotes = models.Quotation.objects.filter(user=user, book=book["id"]).distinct() - # quote_statuses = models.Status.objects.filter( - # id__in=quotes, user=kwargs["user"] - # ).distinct() book["quotes"] = list(quotes.values()) - logger.error("FINAL QUOTES") - logger.error(book["quotes"]) - # append everything final_books.append(book) diff --git a/bookwyrm/models/bookwyrm_import_job.py b/bookwyrm/models/bookwyrm_import_job.py index 696f8061a..6e71aa4b5 100644 --- a/bookwyrm/models/bookwyrm_import_job.py +++ b/bookwyrm/models/bookwyrm_import_job.py @@ -1,5 +1,6 @@ from functools import reduce import json +import logging import operator from django.db.models import FileField, JSONField, CharField @@ -18,7 +19,8 @@ from bookwyrm.models.job import ( create_child_job, ) from bookwyrm.utils.tar import BookwyrmTarFile -import json + +logger = logging.getLogger(__name__) class BookwyrmImportJob(ParentJob): @@ -43,27 +45,33 @@ def start_import_task(**kwargs): if job.complete: return - archive_file.open("rb") - with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: - job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) + try: + archive_file.open("rb") + with BookwyrmTarFile.open(mode="r:gz", fileobj=archive_file) as tar: + job.import_data = json.loads(tar.read("archive.json").decode("utf-8")) - if "include_user_profile" in job.required: - update_user_profile(job.user, tar, job.import_data.get("user")) - if "include_user_settings" in job.required: - update_user_settings(job.user, job.import_data.get("user")) - if "include_goals" in job.required: - update_goals(job.user, job.import_data.get("goals")) - if "include_saved_lists" in job.required: - upsert_saved_lists(job.user, job.import_data.get("saved_lists")) - if "include_follows" in job.required: - upsert_follows(job.user, job.import_data.get("follows")) - if "include_blocks" in job.required: - upsert_user_blocks(job.user, job.import_data.get("blocked_users")) + if "include_user_profile" in job.required: + update_user_profile(job.user, tar, job.import_data.get("user")) + if "include_user_settings" in job.required: + update_user_settings(job.user, job.import_data.get("user")) + if "include_goals" in job.required: + update_goals(job.user, job.import_data.get("goals")) + if "include_saved_lists" in job.required: + upsert_saved_lists(job.user, job.import_data.get("saved_lists")) + if "include_follows" in job.required: + upsert_follows(job.user, job.import_data.get("follows")) + if "include_blocks" in job.required: + upsert_user_blocks(job.user, job.import_data.get("blocked_users")) - process_books(job, tar) + process_books(job, tar) - job.save() - archive_file.close() + job.set_status("complete") # set here to trigger notifications + job.save() + archive_file.close() + + except Exception as err: # pylint: disable=broad-except + logger.exception("User Import Job %s Failed with error: %s", job.id, err) + job.set_status("failed") def process_books(job, tar): diff --git a/bookwyrm/models/job.py b/bookwyrm/models/job.py index 6e8d0dc5c..4ba4bc2d7 100644 --- a/bookwyrm/models/job.py +++ b/bookwyrm/models/job.py @@ -19,6 +19,7 @@ class Job(models.Model): ACTIVE = "active", _("Active") COMPLETE = "complete", _("Complete") STOPPED = "stopped", _("Stopped") + FAILED = "failed", _("Failed") task_id = models.UUIDField(unique=True, null=True, blank=True) @@ -43,14 +44,17 @@ class Job(models.Model): self.save(update_fields=["status", "complete", "updated_date"]) - def stop_job(self): + def stop_job(self, reason=None): """Stop the job""" if self.complete: return self.__terminate_job() - self.status = self.Status.STOPPED + if reason and reason == "failed": + self.status = self.Status.FAILED + else: + self.status = self.Status.STOPPED self.complete = True self.updated_date = timezone.now() @@ -72,6 +76,10 @@ class Job(models.Model): self.stop_job() return + if status == self.Status.FAILED: + self.stop_job(reason="failed") + return + self.updated_date = timezone.now() self.status = status diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index 522038f9a..c8140bce9 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -2,7 +2,15 @@ from django.db import models, transaction from django.dispatch import receiver from .base_model import BookWyrmModel -from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain +from . import ( + Boost, + Favorite, + GroupMemberInvitation, + ImportJob, + BookwyrmImportJob, + LinkDomain, +) +from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from . import ListItem, Report, Status, User, UserFollowRequest @@ -22,6 +30,8 @@ class Notification(BookWyrmModel): # Imports IMPORT = "IMPORT" + USER_IMPORT = "USER_IMPORT" + USER_EXPORT = "USER_EXPORT" # List activity ADD = "ADD" @@ -44,7 +54,7 @@ class Notification(BookWyrmModel): NotificationType = models.TextChoices( # there has got be a better way to do this "NotificationType", - f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}", + f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {USER_IMPORT} {USER_EXPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}", ) user = models.ForeignKey("User", on_delete=models.CASCADE) @@ -61,6 +71,9 @@ class Notification(BookWyrmModel): ) related_status = models.ForeignKey("Status", on_delete=models.CASCADE, null=True) related_import = models.ForeignKey("ImportJob", on_delete=models.CASCADE, null=True) + related_user_export = models.ForeignKey( + "BookwyrmExportJob", on_delete=models.CASCADE, null=True + ) related_list_items = models.ManyToManyField( "ListItem", symmetrical=False, related_name="notifications" ) @@ -223,6 +236,38 @@ def notify_user_on_import_complete( ) +@receiver(models.signals.post_save, sender=BookwyrmImportJob) +# pylint: disable=unused-argument +def notify_user_on_user_import_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we imported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + return + Notification.objects.create( + user=instance.user, notification_type=Notification.USER_IMPORT + ) + + +@receiver(models.signals.post_save, sender=BookwyrmExportJob) +# pylint: disable=unused-argument +def notify_user_on_user_export_complete( + sender, instance, *args, update_fields=None, **kwargs +): + """we imported your user details! aren't you proud of us""" + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: + print("RETURNING", instance.status) + return + print("NOTIFYING") + Notification.objects.create( + user=instance.user, + notification_type=Notification.USER_EXPORT, + related_user_export=instance, + ) + + @receiver(models.signals.post_save, sender=Report) @transaction.atomic # pylint: disable=unused-argument diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index a27c4b70d..cce055999 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -96,6 +96,7 @@ class SiteSettings(SiteModel): imports_enabled = models.BooleanField(default=True) import_size_limit = models.IntegerField(default=0) import_limit_reset = models.IntegerField(default=0) + user_import_time_limit = models.IntegerField(default=48) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) diff --git a/bookwyrm/templates/import/import_user.html b/bookwyrm/templates/import/import_user.html index 86e99f657..8e7bb1a09 100644 --- a/bookwyrm/templates/import/import_user.html +++ b/bookwyrm/templates/import/import_user.html @@ -10,96 +10,89 @@ {% if invalid %}
{% blocktrans %}Currently you are allowed to import one user every {{ user_import_limit_reset }} days.{% endblocktrans %}
-{% blocktrans %}You have {{ allowed_imports }} left.{% endblocktrans %}
-- {% if recent_avg_hours %} - {% blocktrans trimmed with hours=recent_avg_hours|floatformat:0|intcomma %} - On average, recent imports have taken {{ hours }} hours. - {% endblocktrans %} - {% else %} - {% blocktrans trimmed with minutes=recent_avg_minutes|floatformat:0|intcomma %} - On average, recent imports have taken {{ minutes }} minutes. - {% endblocktrans %} - {% endif %} -
+ {% if not site.imports_enabled %} ++ +
++ {% trans "Imports are temporarily disabled; thank you for your patience." %} +
+{% blocktrans %}Currently you are allowed to import one user every {{ user_import_hours }} hours.{% endblocktrans %}
+{% blocktrans %}You will next be able to import a user file at {{ next_available }}{% endblocktrans %}