From 4fa823e8df275a959654a54f82a8109e21793907 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Fri, 29 Mar 2024 22:13:09 +0100 Subject: [PATCH] Update django-storages to 1.14.2 The problem that boto3 closes files has been worked around in django-storages. --- bookwyrm/storage_backends.py | 40 +++------------------------- bookwyrm/views/preferences/export.py | 36 ++++++++++++------------- requirements.txt | 2 +- 3 files changed, 23 insertions(+), 55 deletions(-) diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py index 87c29ae70..1695daa67 100644 --- a/bookwyrm/storage_backends.py +++ b/bookwyrm/storage_backends.py @@ -1,55 +1,23 @@ """Handles backends for storages""" -import os -from tempfile import SpooledTemporaryFile from django.core.files.storage import FileSystemStorage -from storages.backends.s3boto3 import S3Boto3Storage +from storages.backends.s3 import S3Storage from storages.backends.azure_storage import AzureStorage -class StaticStorage(S3Boto3Storage): # pylint: disable=abstract-method +class StaticStorage(S3Storage): # pylint: disable=abstract-method """Storage class for Static contents""" location = "static" default_acl = "public-read" -class ImagesStorage(S3Boto3Storage): # pylint: disable=abstract-method +class ImagesStorage(S3Storage): # pylint: disable=abstract-method """Storage class for Image files""" location = "images" default_acl = "public-read" file_overwrite = False - """ - This is our custom version of S3Boto3Storage that fixes a bug in - boto3 where the passed in file is closed upon upload. - From: - https://github.com/matthewwithanm/django-imagekit/issues/391#issuecomment-275367006 - https://github.com/boto/boto3/issues/929 - https://github.com/matthewwithanm/django-imagekit/issues/391 - """ - - def _save(self, name, content): - """ - We create a clone of the content file as when this is passed to - boto3 it wrongly closes the file upon upload where as the storage - backend expects it to still be open - """ - # Seek our content back to the start - content.seek(0, os.SEEK_SET) - - # Create a temporary file that will write to disk after a specified - # size. This file will be automatically deleted when closed by - # boto3 or after exiting the `with` statement if the boto3 is fixed - with SpooledTemporaryFile() as content_autoclose: - - # Write our original content into our copy that will be closed by boto3 - content_autoclose.write(content.read()) - - # Upload the object which will auto close the - # content_autoclose instance - return super()._save(name, content_autoclose) - class AzureStaticStorage(AzureStorage): # pylint: disable=abstract-method """Storage class for Static contents""" @@ -71,7 +39,7 @@ class ExportsFileStorage(FileSystemStorage): # pylint: disable=abstract-method overwrite_files = False -class ExportsS3Storage(S3Boto3Storage): # pylint: disable=abstract-method +class ExportsS3Storage(S3Storage): # pylint: disable=abstract-method """Storage class for exports contents with S3""" location = "exports" diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index de243586d..58c77b14c 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -14,9 +14,9 @@ from django.urls import reverse from django.utils.decorators import method_decorator from django.shortcuts import redirect -from storages.backends.s3boto3 import S3Boto3Storage +from storages.backends.s3 import S3Storage -from bookwyrm import models, storage_backends +from bookwyrm import models from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm import settings @@ -220,17 +220,16 @@ class ExportUser(View): class ExportArchive(View): """Serve the archive file""" - # TODO: how do we serve s3 files? def get(self, request, archive_id): """download user export file""" export = BookwyrmExportJob.objects.get(task_id=archive_id, user=request.user) - if isinstance(export.export_data.storage, storage_backends.ExportsS3Storage): + if settings.USE_S3: # make custom_domain None so we can sign the url # see https://github.com/jschneier/django-storages/issues/944 - storage = S3Boto3Storage(querystring_auth=True, custom_domain=None) + storage = S3Storage(querystring_auth=True, custom_domain=None) try: - url = S3Boto3Storage.url( + url = S3Storage.url( storage, f"/exports/{export.task_id}.tar.gz", expire=settings.S3_SIGNED_URL_EXPIRY, @@ -239,16 +238,17 @@ class ExportArchive(View): raise Http404() return redirect(url) - if isinstance(export.export_data.storage, storage_backends.ExportsFileStorage): - try: - return HttpResponse( - export.export_data, - content_type="application/gzip", - headers={ - "Content-Disposition": 'attachment; filename="bookwyrm-account-export.tar.gz"' # pylint: disable=line-too-long - }, - ) - except FileNotFoundError: - raise Http404() + if settings.USE_AZURE: + # not implemented + return HttpResponseServerError() - return HttpResponseServerError() + try: + return HttpResponse( + export.export_data, + content_type="application/gzip", + headers={ + "Content-Disposition": 'attachment; filename="bookwyrm-account-export.tar.gz"' # pylint: disable=line-too-long + }, + ) + except FileNotFoundError: + raise Http404() diff --git a/requirements.txt b/requirements.txt index cfda1dfc5..c3b33f94b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ django-oauth-toolkit==2.3.0 django-pgtrigger==4.11.0 django-redis==5.2.0 django-sass-processor==1.2.2 -django-storages==1.13.2 +django-storages==1.14.2 django-storages[azure] environs==11.0.0 flower==2.0.1