Makes expanding book data fully part of the connector

This commit is contained in:
Mouse Reeve 2021-01-02 07:45:45 -08:00
parent fc8f43273e
commit e169565e00
5 changed files with 34 additions and 31 deletions

View file

@ -1,12 +1,10 @@
''' select and call a connector for whatever book task needs doing ''' ''' select and call a connector for whatever book task needs doing '''
import importlib
from urllib.parse import urlparse from urllib.parse import urlparse
from requests import HTTPError from requests import HTTPError
from bookwyrm import models from bookwyrm import models
from bookwyrm.connectors import ConnectorException from bookwyrm.connectors import ConnectorException, load_connector
from bookwyrm.tasks import app
def get_edition(book_id): def get_edition(book_id):
@ -40,14 +38,6 @@ def get_or_create_connector(remote_id):
return load_connector(connector_info) return load_connector(connector_info)
@app.task
def load_more_data(book_id):
''' background the work of getting all 10,000 editions of LoTR '''
book = models.Book.objects.select_subclasses().get(id=book_id)
connector = load_connector(book.connector)
connector.expand_book_data(book)
def search(query, min_confidence=0.1): def search(query, min_confidence=0.1):
''' find books based on arbitary keywords ''' ''' find books based on arbitary keywords '''
results = [] results = []
@ -90,11 +80,3 @@ def get_connectors():
''' load all connectors ''' ''' load all connectors '''
for info in models.Connector.objects.order_by('priority').all(): for info in models.Connector.objects.order_by('priority').all():
yield load_connector(info) yield load_connector(info)
def load_connector(connector_info):
''' instantiate the connector class '''
connector = importlib.import_module(
'bookwyrm.connectors.%s' % connector_info.connector_file
)
return connector.Connector(connector_info.identifier)

View file

@ -1,4 +1,4 @@
''' bring connectors into the namespace ''' ''' bring connectors into the namespace '''
from .settings import CONNECTORS from .settings import CONNECTORS
from .abstract_connector import ConnectorException from .abstract_connector import ConnectorException, load_connector
from .abstract_connector import get_data, get_image from .abstract_connector import get_data, get_image

View file

@ -1,6 +1,7 @@
''' functionality outline for a book data connector ''' ''' functionality outline for a book data connector '''
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
import importlib
import logging import logging
from urllib3.exceptions import RequestError from urllib3.exceptions import RequestError
@ -10,6 +11,7 @@ from requests import HTTPError
from requests.exceptions import SSLError from requests.exceptions import SSLError
from bookwyrm import activitypub, models, settings from bookwyrm import activitypub, models, settings
from bookwyrm.tasks import app
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -90,7 +92,6 @@ class AbstractConnector(AbstractMinimalConnector):
return True return True
@transaction.atomic
def get_or_create_book(self, remote_id): def get_or_create_book(self, remote_id):
''' translate arbitrary json into an Activitypub dataclass ''' ''' translate arbitrary json into an Activitypub dataclass '''
# first, check if we have the origin_id saved # first, check if we have the origin_id saved
@ -123,13 +124,17 @@ class AbstractConnector(AbstractMinimalConnector):
if not work_data or not edition_data: if not work_data or not edition_data:
raise ConnectorException('Unable to load book data: %s' % remote_id) raise ConnectorException('Unable to load book data: %s' % remote_id)
# create activitypub object with transaction.atomic():
work_activity = activitypub.Work(**work_data) # create activitypub object
# this will dedupe automatically work_activity = activitypub.Work(**work_data)
work = work_activity.to_model(models.Work) # this will dedupe automatically
for author in self.get_authors_from_data(data): work = work_activity.to_model(models.Work)
work.authors.add(author) for author in self.get_authors_from_data(data):
return self.create_edition_from_data(work, edition_data) work.authors.add(author)
edition = self.create_edition_from_data(work, edition_data)
load_more_data.delay(self.connector.id, work.id)
return edition
def create_edition_from_data(self, work, edition_data): def create_edition_from_data(self, work, edition_data):
@ -187,6 +192,23 @@ class AbstractConnector(AbstractMinimalConnector):
''' get more info on a book ''' ''' get more info on a book '''
@app.task
def load_more_data(connector_id, book_id):
''' background the work of getting all 10,000 editions of LoTR '''
connector_info = models.Connector.objects.get(id=connector_id)
connector = load_connector(connector_info)
book = models.Book.objects.select_subclasses().get(id=book_id)
connector.expand_book_data(book)
def load_connector(connector_info):
''' instantiate the connector class '''
connector = importlib.import_module(
'bookwyrm.connectors.%s' % connector_info.connector_file
)
return connector.Connector(connector_info.identifier)
def dict_from_mappings(data, mappings): def dict_from_mappings(data, mappings):
''' create a dict in Activitypub format, using mappings supplies by ''' create a dict in Activitypub format, using mappings supplies by
the subclass ''' the subclass '''

View file

@ -212,8 +212,6 @@ def resolve_book(request):
remote_id = request.POST.get('remote_id') remote_id = request.POST.get('remote_id')
connector = books_manager.get_or_create_connector(remote_id) connector = books_manager.get_or_create_connector(remote_id)
book = connector.get_or_create_book(remote_id) book = connector.get_or_create_book(remote_id)
if book.connector:
books_manager.load_more_data.delay(book.id)
return redirect('/book/%d' % book.id) return redirect('/book/%d' % book.id)

View file

@ -20,8 +20,9 @@ app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs. # Load task modules from all registered Django app configs.
app.autodiscover_tasks() app.autodiscover_tasks()
app.autodiscover_tasks(['bookwyrm'], related_name='activitypub.base_activity') app.autodiscover_tasks(['bookwyrm'], related_name='activitypub.base_activity')
app.autodiscover_tasks(['bookwyrm'], related_name='books_manager')
app.autodiscover_tasks(['bookwyrm'], related_name='broadcast') app.autodiscover_tasks(['bookwyrm'], related_name='broadcast')
app.autodiscover_tasks(
['bookwyrm'], related_name='connectors.abstract_connector')
app.autodiscover_tasks(['bookwyrm'], related_name='emailing') app.autodiscover_tasks(['bookwyrm'], related_name='emailing')
app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import') app.autodiscover_tasks(['bookwyrm'], related_name='goodreads_import')
app.autodiscover_tasks(['bookwyrm'], related_name='incoming') app.autodiscover_tasks(['bookwyrm'], related_name='incoming')