diff --git a/bookwyrm/importers/__init__.py b/bookwyrm/importers/__init__.py index dd3d62e8b..6ce50f160 100644 --- a/bookwyrm/importers/__init__.py +++ b/bookwyrm/importers/__init__.py @@ -1,6 +1,7 @@ """ import classes """ from .importer import Importer +from .calibre_import import CalibreImporter from .goodreads_import import GoodreadsImporter from .librarything_import import LibrarythingImporter from .openlibrary_import import OpenLibraryImporter diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py new file mode 100644 index 000000000..7395e2f7b --- /dev/null +++ b/bookwyrm/importers/calibre_import.py @@ -0,0 +1,28 @@ +""" handle reading a csv from calibre """ +from bookwyrm.models import Shelf + +from . import Importer + + +class CalibreImporter(Importer): + """csv downloads from Calibre""" + + service = "Calibre" + + def __init__(self, *args, **kwargs): + # Add timestamp to row_mappings_guesses for date_added to avoid + # integrity error + row_mappings_guesses = [] + + for field, mapping in self.row_mappings_guesses: + if field in ("date_added",): + row_mappings_guesses.append((field, mapping + ["timestamp"])) + else: + row_mappings_guesses.append((field, mapping)) + + self.row_mappings_guesses = row_mappings_guesses + super().__init__(*args, **kwargs) + + def get_shelf(self, normalized_row): + # Calibre export does not indicate which shelf to use. Go with a default one for now + return Shelf.TO_READ diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index 37730dee3..c6833547d 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -1,5 +1,8 @@ """ handle reading a tsv from librarything """ import re + +from bookwyrm.models import Shelf + from . import Importer @@ -21,7 +24,7 @@ class LibrarythingImporter(Importer): def get_shelf(self, normalized_row): if normalized_row["date_finished"]: - return "read" + return Shelf.READ_FINISHED if normalized_row["date_started"]: - return "reading" - return "to-read" + return Shelf.READING + return Shelf.TO_READ diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index bcba391b6..556f133f9 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -175,9 +175,15 @@ class ImportItem(models.Model): def date_added(self): """when the book was added to this dataset""" if self.normalized_data.get("date_added"): - return timezone.make_aware( - dateutil.parser.parse(self.normalized_data.get("date_added")) + parsed_date_added = dateutil.parser.parse( + self.normalized_data.get("date_added") ) + + if timezone.is_aware(parsed_date_added): + # Keep timezone if import already had one + return parsed_date_added + + return timezone.make_aware(parsed_date_added) return None @property diff --git a/bookwyrm/templates/import/import.html b/bookwyrm/templates/import/import.html index 6df7c0843..fc00389c5 100644 --- a/bookwyrm/templates/import/import.html +++ b/bookwyrm/templates/import/import.html @@ -32,6 +32,9 @@ + diff --git a/bookwyrm/tests/data/calibre.csv b/bookwyrm/tests/data/calibre.csv new file mode 100644 index 000000000..4f936cfa9 --- /dev/null +++ b/bookwyrm/tests/data/calibre.csv @@ -0,0 +1,2 @@ +authors,author_sort,rating,library_name,timestamp,formats,size,isbn,identifiers,comments,tags,series,series_index,languages,title,cover,title_sort,publisher,pubdate,id,uuid +"Seanan McGuire","McGuire, Seanan","5","Bücher","2021-01-19T22:41:16+01:00","epub, original_epub","1433809","9780756411800","goodreads:39077187,isbn:9780756411800","REPLACED COMMENTS (BOOK DESCRIPTION) BECAUSE IT IS REALLY LONG.","Cryptids, Fantasy, Romance, Magic","InCryptid","8.0","eng","That Ain't Witchcraft","/home/tastytea/Bücher/Seanan McGuire/That Ain't Witchcraft (864)/cover.jpg","That Ain't Witchcraft","Daw Books","2019-03-05T01:00:00+01:00","864","3051ed45-8943-4900-a22a-d2704e3583df" diff --git a/bookwyrm/tests/importers/test_calibre_import.py b/bookwyrm/tests/importers/test_calibre_import.py new file mode 100644 index 000000000..aff44a241 --- /dev/null +++ b/bookwyrm/tests/importers/test_calibre_import.py @@ -0,0 +1,71 @@ +""" testing import """ +import pathlib +from unittest.mock import patch + +from django.test import TestCase + +from bookwyrm import models +from bookwyrm.importers import CalibreImporter +from bookwyrm.importers.importer import handle_imported_book + + +# pylint: disable=consider-using-with +@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") +@patch("bookwyrm.activitystreams.populate_stream_task.delay") +@patch("bookwyrm.activitystreams.add_book_statuses_task.delay") +class CalibreImport(TestCase): + """importing from Calibre csv""" + + def setUp(self): + """use a test csv""" + self.importer = CalibreImporter() + datafile = pathlib.Path(__file__).parent.joinpath("../data/calibre.csv") + self.csv = open(datafile, "r", encoding=self.importer.encoding) + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"): + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True + ) + + work = models.Work.objects.create(title="Test Work") + self.book = models.Edition.objects.create( + title="Example Edition", + remote_id="https://example.com/book/1", + parent_work=work, + ) + + def test_create_job(self, *_): + """creates the import job entry and checks csv""" + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + + import_items = ( + models.ImportItem.objects.filter(job=import_job).order_by("index").all() + ) + self.assertEqual(len(import_items), 1) + self.assertEqual(import_items[0].index, 0) + self.assertEqual( + import_items[0].normalized_data["title"], "That Ain't Witchcraft" + ) + + def test_handle_imported_book(self, *_): + """calibre import added a book, this adds related connections""" + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.TO_READ + ).first() + self.assertIsNone(shelf.books.first()) + + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + + shelf.refresh_from_db() + self.assertEqual(shelf.books.first(), self.book) diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 04fb886bf..9d1c19085 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -84,7 +84,9 @@ class GoodreadsImport(TestCase): def test_handle_imported_book(self, *_): """goodreads import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index c8da8a271..c12095a4c 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -174,7 +174,9 @@ class GenericImporter(TestCase): def test_handle_imported_book(self, *_): """import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( @@ -193,7 +195,9 @@ class GenericImporter(TestCase): def test_handle_imported_book_already_shelved(self, *_): """import added a book, this adds related connections""" with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - shelf = self.local_user.shelf_set.filter(identifier="to-read").first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.TO_READ + ).first() models.ShelfBook.objects.create( shelf=shelf, user=self.local_user, @@ -217,12 +221,16 @@ class GenericImporter(TestCase): shelf.shelfbook_set.first().shelved_date, make_date(2020, 2, 2) ) self.assertIsNone( - self.local_user.shelf_set.get(identifier="read").books.first() + self.local_user.shelf_set.get( + identifier=models.Shelf.READ_FINISHED + ).books.first() ) def test_handle_import_twice(self, *_): """re-importing books""" - shelf = self.local_user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() import_job = self.importer.create_job( self.local_user, self.csv, False, "public" ) diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 57d555206..3994e8cde 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -93,7 +93,9 @@ class LibrarythingImport(TestCase): def test_handle_imported_book(self, *_): """librarything import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( @@ -117,7 +119,9 @@ class LibrarythingImport(TestCase): def test_handle_imported_book_already_shelved(self, *_): """librarything import added a book, this adds related connections""" with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - shelf = self.local_user.shelf_set.filter(identifier="to-read").first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.TO_READ + ).first() models.ShelfBook.objects.create( shelf=shelf, user=self.local_user, book=self.book ) @@ -135,7 +139,9 @@ class LibrarythingImport(TestCase): shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) self.assertIsNone( - self.local_user.shelf_set.get(identifier="read").books.first() + self.local_user.shelf_set.get( + identifier=models.Shelf.READ_FINISHED + ).books.first() ) readthrough = models.ReadThrough.objects.get(user=self.local_user) diff --git a/bookwyrm/tests/importers/test_openlibrary_import.py b/bookwyrm/tests/importers/test_openlibrary_import.py index a775c5969..28c10e50c 100644 --- a/bookwyrm/tests/importers/test_openlibrary_import.py +++ b/bookwyrm/tests/importers/test_openlibrary_import.py @@ -70,7 +70,9 @@ class OpenLibraryImport(TestCase): def test_handle_imported_book(self, *_): """openlibrary import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="reading").first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READING + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index 670c6e5e4..afff0b218 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -62,7 +62,9 @@ class StorygraphImport(TestCase): def test_handle_imported_book(self, *_): """storygraph import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="to-read").first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.TO_READ + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 6e50a14cc..063545895 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -11,6 +11,7 @@ from django.views import View from bookwyrm import forms, models from bookwyrm.importers import ( + CalibreImporter, LibrarythingImporter, GoodreadsImporter, StorygraphImporter, @@ -52,6 +53,8 @@ class Import(View): importer = StorygraphImporter() elif source == "OpenLibrary": importer = OpenLibraryImporter() + elif source == "Calibre": + importer = CalibreImporter() else: # Default : Goodreads importer = GoodreadsImporter()