From 3626db3c1a517239ad9029026cd7d26c37714b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Jaenisch?= Date: Sat, 30 Apr 2022 15:25:09 +0200 Subject: [PATCH 1/8] Add Calibre importer for CSV exports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Jaenisch --- bookwyrm/importers/__init__.py | 1 + bookwyrm/importers/calibre_import.py | 8 ++++++++ bookwyrm/templates/import/import.html | 3 +++ bookwyrm/views/imports/import_data.py | 3 +++ 4 files changed, 15 insertions(+) create mode 100644 bookwyrm/importers/calibre_import.py diff --git a/bookwyrm/importers/__init__.py b/bookwyrm/importers/__init__.py index dd3d62e8..6ce50f16 100644 --- a/bookwyrm/importers/__init__.py +++ b/bookwyrm/importers/__init__.py @@ -1,6 +1,7 @@ """ import classes """ from .importer import Importer +from .calibre_import import CalibreImporter from .goodreads_import import GoodreadsImporter from .librarything_import import LibrarythingImporter from .openlibrary_import import OpenLibraryImporter diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py new file mode 100644 index 00000000..a0d71263 --- /dev/null +++ b/bookwyrm/importers/calibre_import.py @@ -0,0 +1,8 @@ +""" handle reading a csv from calibre """ +from . import Importer + + +class CalibreImporter(Importer): + """csv downloads from OpenLibrary""" + + service = "Calibre" diff --git a/bookwyrm/templates/import/import.html b/bookwyrm/templates/import/import.html index 6df7c084..fc00389c 100644 --- a/bookwyrm/templates/import/import.html +++ b/bookwyrm/templates/import/import.html @@ -32,6 +32,9 @@ + diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 6e50a14c..06354589 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -11,6 +11,7 @@ from django.views import View from bookwyrm import forms, models from bookwyrm.importers import ( + CalibreImporter, LibrarythingImporter, GoodreadsImporter, StorygraphImporter, @@ -52,6 +53,8 @@ class Import(View): importer = StorygraphImporter() elif source == "OpenLibrary": importer = OpenLibraryImporter() + elif source == "Calibre": + importer = CalibreImporter() else: # Default : Goodreads importer = GoodreadsImporter() From eeb1cc71975d98566ff9ff79f403bc3ec11cd4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Jaenisch?= Date: Sat, 30 Apr 2022 19:08:31 +0200 Subject: [PATCH 2/8] Use a default shelf because Calibre indicates no reading status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Jaenisch --- bookwyrm/importers/calibre_import.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py index a0d71263..18c6358a 100644 --- a/bookwyrm/importers/calibre_import.py +++ b/bookwyrm/importers/calibre_import.py @@ -3,6 +3,10 @@ from . import Importer class CalibreImporter(Importer): - """csv downloads from OpenLibrary""" + """csv downloads from Calibre""" service = "Calibre" + + def get_shelf(self, normalized_row): + # Calibre export does not indicate which shelf to use. Go with a default one for now + return "to-read" From 6bd9b725e27d4ca850275b111b280c8530aa88c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Jaenisch?= Date: Thu, 5 May 2022 13:07:25 +0200 Subject: [PATCH 3/8] Refactor hard-coded strings with a reference to a static property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Jaenisch --- bookwyrm/importers/librarything_import.py | 9 ++++++--- bookwyrm/tests/importers/test_goodreads_import.py | 2 +- bookwyrm/tests/importers/test_importer.py | 8 ++++---- bookwyrm/tests/importers/test_librarything_import.py | 6 +++--- bookwyrm/tests/importers/test_openlibrary_import.py | 2 +- bookwyrm/tests/importers/test_storygraph_import.py | 2 +- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index 37730dee..c6833547 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -1,5 +1,8 @@ """ handle reading a tsv from librarything """ import re + +from bookwyrm.models import Shelf + from . import Importer @@ -21,7 +24,7 @@ class LibrarythingImporter(Importer): def get_shelf(self, normalized_row): if normalized_row["date_finished"]: - return "read" + return Shelf.READ_FINISHED if normalized_row["date_started"]: - return "reading" - return "to-read" + return Shelf.READING + return Shelf.TO_READ diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 04fb886b..00c8b981 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -84,7 +84,7 @@ class GoodreadsImport(TestCase): def test_handle_imported_book(self, *_): """goodreads import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READ_FINISHED).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index c8da8a27..5d0b9b00 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -174,7 +174,7 @@ class GenericImporter(TestCase): def test_handle_imported_book(self, *_): """import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READ_FINISHED).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( @@ -193,7 +193,7 @@ class GenericImporter(TestCase): def test_handle_imported_book_already_shelved(self, *_): """import added a book, this adds related connections""" with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - shelf = self.local_user.shelf_set.filter(identifier="to-read").first() + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.TO_READ).first() models.ShelfBook.objects.create( shelf=shelf, user=self.local_user, @@ -217,12 +217,12 @@ class GenericImporter(TestCase): shelf.shelfbook_set.first().shelved_date, make_date(2020, 2, 2) ) self.assertIsNone( - self.local_user.shelf_set.get(identifier="read").books.first() + self.local_user.shelf_set.get(identifier=models.Shelf.READ_FINISHED).books.first() ) def test_handle_import_twice(self, *_): """re-importing books""" - shelf = self.local_user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READ_FINISHED).first() import_job = self.importer.create_job( self.local_user, self.csv, False, "public" ) diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 57d55520..e976027c 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -93,7 +93,7 @@ class LibrarythingImport(TestCase): def test_handle_imported_book(self, *_): """librarything import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READ_FINISHED).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( @@ -117,7 +117,7 @@ class LibrarythingImport(TestCase): def test_handle_imported_book_already_shelved(self, *_): """librarything import added a book, this adds related connections""" with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - shelf = self.local_user.shelf_set.filter(identifier="to-read").first() + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.TO_READ).first() models.ShelfBook.objects.create( shelf=shelf, user=self.local_user, book=self.book ) @@ -135,7 +135,7 @@ class LibrarythingImport(TestCase): shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) self.assertIsNone( - self.local_user.shelf_set.get(identifier="read").books.first() + self.local_user.shelf_set.get(identifier=models.Shelf.READ_FINISHED).books.first() ) readthrough = models.ReadThrough.objects.get(user=self.local_user) diff --git a/bookwyrm/tests/importers/test_openlibrary_import.py b/bookwyrm/tests/importers/test_openlibrary_import.py index a775c596..dbd177b8 100644 --- a/bookwyrm/tests/importers/test_openlibrary_import.py +++ b/bookwyrm/tests/importers/test_openlibrary_import.py @@ -70,7 +70,7 @@ class OpenLibraryImport(TestCase): def test_handle_imported_book(self, *_): """openlibrary import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="reading").first() + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READING).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index 670c6e5e..69788d60 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -62,7 +62,7 @@ class StorygraphImport(TestCase): def test_handle_imported_book(self, *_): """storygraph import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier="to-read").first() + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.TO_READ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( From 22fcb61fb23654028765431dda4405cd6d132869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Jaenisch?= Date: Thu, 5 May 2022 13:08:01 +0200 Subject: [PATCH 4/8] Write tests for Calibre importer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Jaenisch --- bookwyrm/importers/calibre_import.py | 4 +- bookwyrm/tests/data/calibre.csv | 2 + .../tests/importers/test_calibre_import.py | 67 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 bookwyrm/tests/data/calibre.csv create mode 100644 bookwyrm/tests/importers/test_calibre_import.py diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py index 18c6358a..ce20fbfa 100644 --- a/bookwyrm/importers/calibre_import.py +++ b/bookwyrm/importers/calibre_import.py @@ -1,4 +1,6 @@ """ handle reading a csv from calibre """ +from bookwyrm.models import Shelf + from . import Importer @@ -9,4 +11,4 @@ class CalibreImporter(Importer): def get_shelf(self, normalized_row): # Calibre export does not indicate which shelf to use. Go with a default one for now - return "to-read" + return Shelf.TO_READ diff --git a/bookwyrm/tests/data/calibre.csv b/bookwyrm/tests/data/calibre.csv new file mode 100644 index 00000000..4f936cfa --- /dev/null +++ b/bookwyrm/tests/data/calibre.csv @@ -0,0 +1,2 @@ +authors,author_sort,rating,library_name,timestamp,formats,size,isbn,identifiers,comments,tags,series,series_index,languages,title,cover,title_sort,publisher,pubdate,id,uuid +"Seanan McGuire","McGuire, Seanan","5","Bücher","2021-01-19T22:41:16+01:00","epub, original_epub","1433809","9780756411800","goodreads:39077187,isbn:9780756411800","REPLACED COMMENTS (BOOK DESCRIPTION) BECAUSE IT IS REALLY LONG.","Cryptids, Fantasy, Romance, Magic","InCryptid","8.0","eng","That Ain't Witchcraft","/home/tastytea/Bücher/Seanan McGuire/That Ain't Witchcraft (864)/cover.jpg","That Ain't Witchcraft","Daw Books","2019-03-05T01:00:00+01:00","864","3051ed45-8943-4900-a22a-d2704e3583df" diff --git a/bookwyrm/tests/importers/test_calibre_import.py b/bookwyrm/tests/importers/test_calibre_import.py new file mode 100644 index 00000000..5257caa6 --- /dev/null +++ b/bookwyrm/tests/importers/test_calibre_import.py @@ -0,0 +1,67 @@ +""" testing import """ +import pathlib +from unittest.mock import patch + +from django.test import TestCase + +from bookwyrm import models +from bookwyrm.importers import CalibreImporter +from bookwyrm.importers.importer import handle_imported_book + + +# pylint: disable=consider-using-with +@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") +@patch("bookwyrm.activitystreams.populate_stream_task.delay") +@patch("bookwyrm.activitystreams.add_book_statuses_task.delay") +class CalibreImport(TestCase): + """importing from Calibre csv""" + + def setUp(self): + """use a test csv""" + self.importer = CalibreImporter() + datafile = pathlib.Path(__file__).parent.joinpath("../data/calibre.csv") + self.csv = open(datafile, "r", encoding=self.importer.encoding) + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ), patch("bookwyrm.lists_stream.populate_lists_task.delay"): + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True + ) + + work = models.Work.objects.create(title="Test Work") + self.book = models.Edition.objects.create( + title="Example Edition", + remote_id="https://example.com/book/1", + parent_work=work, + ) + + def test_create_job(self, *_): + """creates the import job entry and checks csv""" + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + + import_items = ( + models.ImportItem.objects.filter(job=import_job).order_by("index").all() + ) + self.assertEqual(len(import_items), 1) + self.assertEqual(import_items[0].index, 0) + self.assertEqual(import_items[0].normalized_data["title"], "That Ain't Witchcraft") + + def test_handle_imported_book(self, *_): + """calibre import added a book, this adds related connections""" + shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.TO_READ).first() + self.assertIsNone(shelf.books.first()) + + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + + shelf.refresh_from_db() + self.assertEqual(shelf.books.first(), self.book) From 62c7661fb92c282d72a0d1ff5bc8509bd1635ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Jaenisch?= Date: Thu, 5 May 2022 21:31:56 +0200 Subject: [PATCH 5/8] Reformat tests using black MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Jaenisch --- bookwyrm/tests/importers/test_calibre_import.py | 8 ++++++-- .../tests/importers/test_goodreads_import.py | 4 +++- bookwyrm/tests/importers/test_importer.py | 16 ++++++++++++---- .../tests/importers/test_librarything_import.py | 12 +++++++++--- .../tests/importers/test_openlibrary_import.py | 4 +++- .../tests/importers/test_storygraph_import.py | 4 +++- 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/bookwyrm/tests/importers/test_calibre_import.py b/bookwyrm/tests/importers/test_calibre_import.py index 5257caa6..aff44a24 100644 --- a/bookwyrm/tests/importers/test_calibre_import.py +++ b/bookwyrm/tests/importers/test_calibre_import.py @@ -46,11 +46,15 @@ class CalibreImport(TestCase): ) self.assertEqual(len(import_items), 1) self.assertEqual(import_items[0].index, 0) - self.assertEqual(import_items[0].normalized_data["title"], "That Ain't Witchcraft") + self.assertEqual( + import_items[0].normalized_data["title"], "That Ain't Witchcraft" + ) def test_handle_imported_book(self, *_): """calibre import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.TO_READ).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.TO_READ + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 00c8b981..9d1c1908 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -84,7 +84,9 @@ class GoodreadsImport(TestCase): def test_handle_imported_book(self, *_): """goodreads import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READ_FINISHED).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index 5d0b9b00..c12095a4 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -174,7 +174,9 @@ class GenericImporter(TestCase): def test_handle_imported_book(self, *_): """import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READ_FINISHED).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( @@ -193,7 +195,9 @@ class GenericImporter(TestCase): def test_handle_imported_book_already_shelved(self, *_): """import added a book, this adds related connections""" with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.TO_READ).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.TO_READ + ).first() models.ShelfBook.objects.create( shelf=shelf, user=self.local_user, @@ -217,12 +221,16 @@ class GenericImporter(TestCase): shelf.shelfbook_set.first().shelved_date, make_date(2020, 2, 2) ) self.assertIsNone( - self.local_user.shelf_set.get(identifier=models.Shelf.READ_FINISHED).books.first() + self.local_user.shelf_set.get( + identifier=models.Shelf.READ_FINISHED + ).books.first() ) def test_handle_import_twice(self, *_): """re-importing books""" - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READ_FINISHED).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() import_job = self.importer.create_job( self.local_user, self.csv, False, "public" ) diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index e976027c..3994e8cd 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -93,7 +93,9 @@ class LibrarythingImport(TestCase): def test_handle_imported_book(self, *_): """librarything import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READ_FINISHED).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READ_FINISHED + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( @@ -117,7 +119,9 @@ class LibrarythingImport(TestCase): def test_handle_imported_book_already_shelved(self, *_): """librarything import added a book, this adds related connections""" with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.TO_READ).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.TO_READ + ).first() models.ShelfBook.objects.create( shelf=shelf, user=self.local_user, book=self.book ) @@ -135,7 +139,9 @@ class LibrarythingImport(TestCase): shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) self.assertIsNone( - self.local_user.shelf_set.get(identifier=models.Shelf.READ_FINISHED).books.first() + self.local_user.shelf_set.get( + identifier=models.Shelf.READ_FINISHED + ).books.first() ) readthrough = models.ReadThrough.objects.get(user=self.local_user) diff --git a/bookwyrm/tests/importers/test_openlibrary_import.py b/bookwyrm/tests/importers/test_openlibrary_import.py index dbd177b8..28c10e50 100644 --- a/bookwyrm/tests/importers/test_openlibrary_import.py +++ b/bookwyrm/tests/importers/test_openlibrary_import.py @@ -70,7 +70,9 @@ class OpenLibraryImport(TestCase): def test_handle_imported_book(self, *_): """openlibrary import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.READING).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.READING + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index 69788d60..afff0b21 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -62,7 +62,9 @@ class StorygraphImport(TestCase): def test_handle_imported_book(self, *_): """storygraph import added a book, this adds related connections""" - shelf = self.local_user.shelf_set.filter(identifier=models.Shelf.TO_READ).first() + shelf = self.local_user.shelf_set.filter( + identifier=models.Shelf.TO_READ + ).first() self.assertIsNone(shelf.books.first()) import_job = self.importer.create_job( From 12541d5f1c46e7e074b51d19b2b8fd90b9f3e7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Jaenisch?= Date: Mon, 23 May 2022 20:52:26 +0200 Subject: [PATCH 6/8] Map timestamp to date_added to avoid integrity error. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Jaenisch --- bookwyrm/importers/calibre_import.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py index ce20fbfa..b9eb3da7 100644 --- a/bookwyrm/importers/calibre_import.py +++ b/bookwyrm/importers/calibre_import.py @@ -9,6 +9,20 @@ class CalibreImporter(Importer): service = "Calibre" + def __init__(self, *args, **kwargs): + # Add timestamp to row_mappings_guesses for date_added to avoid + # integrity error + row_mappings_guesses = [] + + for field, mapping in self.row_mappings_guesses: + if field in ('date_added',): + row_mappings_guesses.append((field, mapping + ['timestamp'])) + else: + row_mappings_guesses.append((field, mapping)) + + self.row_mappings_guesses = row_mappings_guesses + super().__init__(*args, **kwargs) + def get_shelf(self, normalized_row): # Calibre export does not indicate which shelf to use. Go with a default one for now return Shelf.TO_READ From b564e514fdc1afc35ececdaf1e45b384e8d2c6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Jaenisch?= Date: Mon, 23 May 2022 20:52:57 +0200 Subject: [PATCH 7/8] Handle parsed dates that already have a timezone on import. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Jaenisch --- bookwyrm/models/import_job.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index bcba391b..7ebac0e1 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -175,9 +175,12 @@ class ImportItem(models.Model): def date_added(self): """when the book was added to this dataset""" if self.normalized_data.get("date_added"): - return timezone.make_aware( - dateutil.parser.parse(self.normalized_data.get("date_added")) - ) + parsed_date_added = dateutil.parser.parse(self.normalized_data.get("date_added")) + if timezone.is_aware(parsed_date_added): + # Keep timezone if import already had one + return parsed_date_added + + return timezone.make_aware(parsed_date_added) return None @property From d837146b660334845f123dbef52264253c02fc9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Jaenisch?= Date: Mon, 23 May 2022 20:59:28 +0200 Subject: [PATCH 8/8] Make black happy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Jaenisch --- bookwyrm/importers/calibre_import.py | 4 ++-- bookwyrm/models/import_job.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py index b9eb3da7..7395e2f7 100644 --- a/bookwyrm/importers/calibre_import.py +++ b/bookwyrm/importers/calibre_import.py @@ -15,8 +15,8 @@ class CalibreImporter(Importer): row_mappings_guesses = [] for field, mapping in self.row_mappings_guesses: - if field in ('date_added',): - row_mappings_guesses.append((field, mapping + ['timestamp'])) + if field in ("date_added",): + row_mappings_guesses.append((field, mapping + ["timestamp"])) else: row_mappings_guesses.append((field, mapping)) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 7ebac0e1..556f133f 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -175,7 +175,10 @@ class ImportItem(models.Model): def date_added(self): """when the book was added to this dataset""" if self.normalized_data.get("date_added"): - parsed_date_added = dateutil.parser.parse(self.normalized_data.get("date_added")) + parsed_date_added = dateutil.parser.parse( + self.normalized_data.get("date_added") + ) + if timezone.is_aware(parsed_date_added): # Keep timezone if import already had one return parsed_date_added