Merge pull request #2093 from Ryuno-Ki/calibre-import

Calibre import. Fixes #627
This commit is contained in:
Mouse Reeve 2022-05-23 12:37:50 -07:00 committed by GitHub
commit 1843959d10
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 152 additions and 15 deletions

View file

@ -1,6 +1,7 @@
""" import classes """
from .importer import Importer
from .calibre_import import CalibreImporter
from .goodreads_import import GoodreadsImporter
from .librarything_import import LibrarythingImporter
from .openlibrary_import import OpenLibraryImporter

View file

@ -0,0 +1,28 @@
""" handle reading a csv from calibre """
from bookwyrm.models import Shelf
from . import Importer
class CalibreImporter(Importer):
"""csv downloads from Calibre"""
service = "Calibre"
def __init__(self, *args, **kwargs):
# Add timestamp to row_mappings_guesses for date_added to avoid
# integrity error
row_mappings_guesses = []
for field, mapping in self.row_mappings_guesses:
if field in ("date_added",):
row_mappings_guesses.append((field, mapping + ["timestamp"]))
else:
row_mappings_guesses.append((field, mapping))
self.row_mappings_guesses = row_mappings_guesses
super().__init__(*args, **kwargs)
def get_shelf(self, normalized_row):
# Calibre export does not indicate which shelf to use. Go with a default one for now
return Shelf.TO_READ

View file

@ -1,5 +1,8 @@
""" handle reading a tsv from librarything """
import re
from bookwyrm.models import Shelf
from . import Importer
@ -21,7 +24,7 @@ class LibrarythingImporter(Importer):
def get_shelf(self, normalized_row):
if normalized_row["date_finished"]:
return "read"
return Shelf.READ_FINISHED
if normalized_row["date_started"]:
return "reading"
return "to-read"
return Shelf.READING
return Shelf.TO_READ

View file

@ -175,9 +175,15 @@ class ImportItem(models.Model):
def date_added(self):
"""when the book was added to this dataset"""
if self.normalized_data.get("date_added"):
return timezone.make_aware(
dateutil.parser.parse(self.normalized_data.get("date_added"))
parsed_date_added = dateutil.parser.parse(
self.normalized_data.get("date_added")
)
if timezone.is_aware(parsed_date_added):
# Keep timezone if import already had one
return parsed_date_added
return timezone.make_aware(parsed_date_added)
return None
@property

View file

@ -32,6 +32,9 @@
<option value="OpenLibrary" {% if current == 'OpenLibrary' %}selected{% endif %}>
OpenLibrary (CSV)
</option>
<option value="Calibre" {% if current == 'Calibre' %}selected{% endif %}>
Calibre (CSV)
</option>
</select>
</div>

View file

@ -0,0 +1,2 @@
authors,author_sort,rating,library_name,timestamp,formats,size,isbn,identifiers,comments,tags,series,series_index,languages,title,cover,title_sort,publisher,pubdate,id,uuid
"Seanan McGuire","McGuire, Seanan","5","Bücher","2021-01-19T22:41:16+01:00","epub, original_epub","1433809","9780756411800","goodreads:39077187,isbn:9780756411800","REPLACED COMMENTS (BOOK DESCRIPTION) BECAUSE IT IS REALLY LONG.","Cryptids, Fantasy, Romance, Magic","InCryptid","8.0","eng","That Ain't Witchcraft","/home/tastytea/Bücher/Seanan McGuire/That Ain't Witchcraft (864)/cover.jpg","That Ain't Witchcraft","Daw Books","2019-03-05T01:00:00+01:00","864","3051ed45-8943-4900-a22a-d2704e3583df"
1 authors author_sort rating library_name timestamp formats size isbn identifiers comments tags series series_index languages title cover title_sort publisher pubdate id uuid
2 Seanan McGuire McGuire, Seanan 5 Bücher 2021-01-19T22:41:16+01:00 epub, original_epub 1433809 9780756411800 goodreads:39077187,isbn:9780756411800 REPLACED COMMENTS (BOOK DESCRIPTION) BECAUSE IT IS REALLY LONG. Cryptids, Fantasy, Romance, Magic InCryptid 8.0 eng That Ain't Witchcraft /home/tastytea/Bücher/Seanan McGuire/That Ain't Witchcraft (864)/cover.jpg That Ain't Witchcraft Daw Books 2019-03-05T01:00:00+01:00 864 3051ed45-8943-4900-a22a-d2704e3583df

View file

@ -0,0 +1,71 @@
""" testing import """
import pathlib
from unittest.mock import patch
from django.test import TestCase
from bookwyrm import models
from bookwyrm.importers import CalibreImporter
from bookwyrm.importers.importer import handle_imported_book
# pylint: disable=consider-using-with
@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
@patch("bookwyrm.activitystreams.populate_stream_task.delay")
@patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
class CalibreImport(TestCase):
"""importing from Calibre csv"""
def setUp(self):
"""use a test csv"""
self.importer = CalibreImporter()
datafile = pathlib.Path(__file__).parent.joinpath("../data/calibre.csv")
self.csv = open(datafile, "r", encoding=self.importer.encoding)
with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
"bookwyrm.activitystreams.populate_stream_task.delay"
), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
self.local_user = models.User.objects.create_user(
"mouse", "mouse@mouse.mouse", "password", local=True
)
work = models.Work.objects.create(title="Test Work")
self.book = models.Edition.objects.create(
title="Example Edition",
remote_id="https://example.com/book/1",
parent_work=work,
)
def test_create_job(self, *_):
"""creates the import job entry and checks csv"""
import_job = self.importer.create_job(
self.local_user, self.csv, False, "public"
)
import_items = (
models.ImportItem.objects.filter(job=import_job).order_by("index").all()
)
self.assertEqual(len(import_items), 1)
self.assertEqual(import_items[0].index, 0)
self.assertEqual(
import_items[0].normalized_data["title"], "That Ain't Witchcraft"
)
def test_handle_imported_book(self, *_):
"""calibre import added a book, this adds related connections"""
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.TO_READ
).first()
self.assertIsNone(shelf.books.first())
import_job = self.importer.create_job(
self.local_user, self.csv, False, "public"
)
import_item = import_job.items.first()
import_item.book = self.book
import_item.save()
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
handle_imported_book(import_item)
shelf.refresh_from_db()
self.assertEqual(shelf.books.first(), self.book)

View file

@ -84,7 +84,9 @@ class GoodreadsImport(TestCase):
def test_handle_imported_book(self, *_):
"""goodreads import added a book, this adds related connections"""
shelf = self.local_user.shelf_set.filter(identifier="read").first()
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.READ_FINISHED
).first()
self.assertIsNone(shelf.books.first())
import_job = self.importer.create_job(

View file

@ -174,7 +174,9 @@ class GenericImporter(TestCase):
def test_handle_imported_book(self, *_):
"""import added a book, this adds related connections"""
shelf = self.local_user.shelf_set.filter(identifier="read").first()
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.READ_FINISHED
).first()
self.assertIsNone(shelf.books.first())
import_job = self.importer.create_job(
@ -193,7 +195,9 @@ class GenericImporter(TestCase):
def test_handle_imported_book_already_shelved(self, *_):
"""import added a book, this adds related connections"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
shelf = self.local_user.shelf_set.filter(identifier="to-read").first()
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.TO_READ
).first()
models.ShelfBook.objects.create(
shelf=shelf,
user=self.local_user,
@ -217,12 +221,16 @@ class GenericImporter(TestCase):
shelf.shelfbook_set.first().shelved_date, make_date(2020, 2, 2)
)
self.assertIsNone(
self.local_user.shelf_set.get(identifier="read").books.first()
self.local_user.shelf_set.get(
identifier=models.Shelf.READ_FINISHED
).books.first()
)
def test_handle_import_twice(self, *_):
"""re-importing books"""
shelf = self.local_user.shelf_set.filter(identifier="read").first()
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.READ_FINISHED
).first()
import_job = self.importer.create_job(
self.local_user, self.csv, False, "public"
)

View file

@ -93,7 +93,9 @@ class LibrarythingImport(TestCase):
def test_handle_imported_book(self, *_):
"""librarything import added a book, this adds related connections"""
shelf = self.local_user.shelf_set.filter(identifier="read").first()
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.READ_FINISHED
).first()
self.assertIsNone(shelf.books.first())
import_job = self.importer.create_job(
@ -117,7 +119,9 @@ class LibrarythingImport(TestCase):
def test_handle_imported_book_already_shelved(self, *_):
"""librarything import added a book, this adds related connections"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
shelf = self.local_user.shelf_set.filter(identifier="to-read").first()
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.TO_READ
).first()
models.ShelfBook.objects.create(
shelf=shelf, user=self.local_user, book=self.book
)
@ -135,7 +139,9 @@ class LibrarythingImport(TestCase):
shelf.refresh_from_db()
self.assertEqual(shelf.books.first(), self.book)
self.assertIsNone(
self.local_user.shelf_set.get(identifier="read").books.first()
self.local_user.shelf_set.get(
identifier=models.Shelf.READ_FINISHED
).books.first()
)
readthrough = models.ReadThrough.objects.get(user=self.local_user)

View file

@ -70,7 +70,9 @@ class OpenLibraryImport(TestCase):
def test_handle_imported_book(self, *_):
"""openlibrary import added a book, this adds related connections"""
shelf = self.local_user.shelf_set.filter(identifier="reading").first()
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.READING
).first()
self.assertIsNone(shelf.books.first())
import_job = self.importer.create_job(

View file

@ -62,7 +62,9 @@ class StorygraphImport(TestCase):
def test_handle_imported_book(self, *_):
"""storygraph import added a book, this adds related connections"""
shelf = self.local_user.shelf_set.filter(identifier="to-read").first()
shelf = self.local_user.shelf_set.filter(
identifier=models.Shelf.TO_READ
).first()
self.assertIsNone(shelf.books.first())
import_job = self.importer.create_job(

View file

@ -11,6 +11,7 @@ from django.views import View
from bookwyrm import forms, models
from bookwyrm.importers import (
CalibreImporter,
LibrarythingImporter,
GoodreadsImporter,
StorygraphImporter,
@ -52,6 +53,8 @@ class Import(View):
importer = StorygraphImporter()
elif source == "OpenLibrary":
importer = OpenLibraryImporter()
elif source == "Calibre":
importer = CalibreImporter()
else:
# Default : Goodreads
importer = GoodreadsImporter()