Merge pull request #2032 from viviicat/bw-dev-npm-fix

Conflicts:
	bw-dev
	dev-tools/Dockerfile
	bookwyrm/static/css/bookwyrm/_all.scss
	bookwyrm/static/css/themes/bookwyrm-dark.scss
	bookwyrm/static/css/themes/bookwyrm-light.scss
This commit is contained in:
Adeodato Simó 2023-10-18 17:06:47 -03:00
commit 6392a8e01d
No known key found for this signature in database
GPG key ID: CDF447845F1A986F
440 changed files with 57032 additions and 12152 deletions

View file

@ -5,3 +5,4 @@ __pycache__
.git .git
.github .github
.pytest* .pytest*
.env

View file

@ -8,7 +8,7 @@ USE_HTTPS=true
DOMAIN=your.domain.here DOMAIN=your.domain.here
EMAIL=your@email.here EMAIL=your@email.here
# Instance defualt language (see options at bookwyrm/settings.py "LANGUAGES" # Instance default language (see options at bookwyrm/settings.py "LANGUAGES"
LANGUAGE_CODE="en-us" LANGUAGE_CODE="en-us"
# Used for deciding which editions to prefer # Used for deciding which editions to prefer
DEFAULT_LANGUAGE="English" DEFAULT_LANGUAGE="English"
@ -21,8 +21,8 @@ MEDIA_ROOT=images/
# Database configuration # Database configuration
PGPORT=5432 PGPORT=5432
POSTGRES_PASSWORD=securedbypassword123 POSTGRES_PASSWORD=securedbypassword123
POSTGRES_USER=fedireads POSTGRES_USER=bookwyrm
POSTGRES_DB=fedireads POSTGRES_DB=bookwyrm
POSTGRES_HOST=db POSTGRES_HOST=db
# Redis activity stream manager # Redis activity stream manager
@ -32,12 +32,17 @@ REDIS_ACTIVITY_PORT=6379
REDIS_ACTIVITY_PASSWORD=redispassword345 REDIS_ACTIVITY_PASSWORD=redispassword345
# Optional, use a different redis database (defaults to 0) # Optional, use a different redis database (defaults to 0)
# REDIS_ACTIVITY_DB_INDEX=0 # REDIS_ACTIVITY_DB_INDEX=0
# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
# REDIS_ACTIVITY_URL=
# Redis as celery broker # Redis as celery broker
REDIS_BROKER_HOST=redis_broker
REDIS_BROKER_PORT=6379 REDIS_BROKER_PORT=6379
REDIS_BROKER_PASSWORD=redispassword123 REDIS_BROKER_PASSWORD=redispassword123
# Optional, use a different redis database (defaults to 0) # Optional, use a different redis database (defaults to 0)
# REDIS_BROKER_DB_INDEX=0 # REDIS_BROKER_DB_INDEX=0
# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
# REDIS_BROKER_URL=
# Monitoring for celery # Monitoring for celery
FLOWER_PORT=8888 FLOWER_PORT=8888
@ -60,7 +65,7 @@ SEARCH_TIMEOUT=5
QUERY_TIMEOUT=5 QUERY_TIMEOUT=5
# Thumbnails Generation # Thumbnails Generation
ENABLE_THUMBNAIL_GENERATION=false ENABLE_THUMBNAIL_GENERATION=true
# S3 configuration # S3 configuration
USE_S3=false USE_S3=false
@ -77,9 +82,15 @@ AWS_SECRET_ACCESS_KEY=
# AWS_S3_REGION_NAME=None # "fr-par" # AWS_S3_REGION_NAME=None # "fr-par"
# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud" # AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud"
# Commented are example values if you use Azure Blob Storage
# USE_AZURE=true
# AZURE_ACCOUNT_NAME= # "example-account-name"
# AZURE_ACCOUNT_KEY= # "base64-encoded-access-key"
# AZURE_CONTAINER= # "example-blob-container-name"
# AZURE_CUSTOM_DOMAIN= # "example-account-name.blob.core.windows.net"
# Preview image generation can be computing and storage intensive # Preview image generation can be computing and storage intensive
# ENABLE_PREVIEW_IMAGES=True ENABLE_PREVIEW_IMAGES=False
# Specify RGB tuple or RGB hex strings, # Specify RGB tuple or RGB hex strings,
# or use_dominant_color_light / use_dominant_color_dark # or use_dominant_color_light / use_dominant_color_dark
@ -108,3 +119,21 @@ OTEL_EXPORTER_OTLP_ENDPOINT=
OTEL_EXPORTER_OTLP_HEADERS= OTEL_EXPORTER_OTLP_HEADERS=
# Service name to identify your app # Service name to identify your app
OTEL_SERVICE_NAME= OTEL_SERVICE_NAME=
# Set HTTP_X_FORWARDED_PROTO ONLY to true if you know what you are doing.
# Only use it if your proxy is "swallowing" if the original request was made
# via https. Please refer to the Django-Documentation and assess the risks
# for your instance:
# https://docs.djangoproject.com/en/3.2/ref/settings/#secure-proxy-ssl-header
HTTP_X_FORWARDED_PROTO=false
# TOTP settings
# TWO_FACTOR_LOGIN_VALIDITY_WINDOW sets the number of codes either side
# which will be accepted.
TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
TWO_FACTOR_LOGIN_MAX_SECONDS=60
# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
# Value should be a comma-separated list of host names.
CSP_ADDITIONAL_HOSTS=

View file

@ -10,6 +10,8 @@ jobs:
lint: lint:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- uses: actions/setup-python@v2 - uses: actions/setup-python@v4
- uses: psf/black@21.4b2 - uses: psf/black@22.12.0
with:
version: 22.12.0

View file

@ -36,11 +36,11 @@ jobs:
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v2 uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning. # Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL - name: Initialize CodeQL
uses: github/codeql-action/init@v1 uses: github/codeql-action/init@v2
with: with:
languages: ${{ matrix.language }} languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file. # If you wish to specify custom queries, you can do so here or in a config file.
@ -51,7 +51,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below) # If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild - name: Autobuild
uses: github/codeql-action/autobuild@v1 uses: github/codeql-action/autobuild@v2
# Command-line programs to run using the OS shell. # Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl # 📚 https://git.io/JvXDl
@ -65,4 +65,4 @@ jobs:
# make release # make release
- name: Perform CodeQL Analysis - name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1 uses: github/codeql-action/analyze@v2

View file

@ -10,7 +10,7 @@ jobs:
lint: lint:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Install curlylint - name: Install curlylint
run: pip install curlylint run: pip install curlylint

View file

@ -23,9 +23,9 @@ jobs:
ports: ports:
- 5432:5432 - 5432:5432
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v2 uses: actions/setup-python@v4
with: with:
python-version: 3.9 python-version: 3.9
- name: Install Dependencies - name: Install Dependencies
@ -56,5 +56,6 @@ jobs:
EMAIL_USE_TLS: true EMAIL_USE_TLS: true
ENABLE_PREVIEW_IMAGES: false ENABLE_PREVIEW_IMAGES: false
ENABLE_THUMBNAIL_GENERATION: true ENABLE_THUMBNAIL_GENERATION: true
HTTP_X_FORWARDED_PROTO: false
run: | run: |
pytest -n 3 pytest -n 3

View file

@ -19,7 +19,7 @@ jobs:
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Install modules - name: Install modules
run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint

50
.github/workflows/mypy.yml vendored Normal file
View file

@ -0,0 +1,50 @@
name: Mypy
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install Dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Analysing the code with mypy
env:
SECRET_KEY: beepbeep
DEBUG: false
USE_HTTPS: true
DOMAIN: your.domain.here
BOOKWYRM_DATABASE_BACKEND: postgres
MEDIA_ROOT: images/
POSTGRES_PASSWORD: hunter2
POSTGRES_USER: postgres
POSTGRES_DB: github_actions
POSTGRES_HOST: 127.0.0.1
CELERY_BROKER: ""
REDIS_BROKER_PORT: 6379
REDIS_BROKER_PASSWORD: beep
USE_DUMMY_CACHE: true
FLOWER_PORT: 8888
EMAIL_HOST: "smtp.mailgun.org"
EMAIL_PORT: 587
EMAIL_HOST_USER: ""
EMAIL_HOST_PASSWORD: ""
EMAIL_USE_TLS: true
ENABLE_PREVIEW_IMAGES: false
ENABLE_THUMBNAIL_GENERATION: true
HTTP_X_FORWARDED_PROTO: false
run: |
mypy bookwyrm celerywyrm

View file

@ -14,10 +14,10 @@ jobs:
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Install modules - name: Install modules
run: npm install prettier run: npm install prettier@2.5.1
- name: Run Prettier - name: Run Prettier
run: npx prettier --check bookwyrm/static/js/*.js run: npx prettier --check bookwyrm/static/js/*.js

View file

@ -12,9 +12,9 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Set up Python 3.9 - name: Set up Python 3.9
uses: actions/setup-python@v2 uses: actions/setup-python@v4
with: with:
python-version: 3.9 python-version: 3.9
- name: Install Dependencies - name: Install Dependencies

1
.prettierrc Normal file
View file

@ -0,0 +1 @@
'trailingComma': 'es5'

333
FEDERATION.md Normal file
View file

@ -0,0 +1,333 @@
# Federation
BookWyrm uses the [ActivityPub](http://activitypub.rocks/) protocol to send and receive user activity between other BookWyrm instances and other services that implement ActivityPub. To handle book data, BookWyrm has a handful of extended Activity types which are not part of the standard, but are legible to other BookWyrm instances.
## Activities and Objects
### Users and relationships
User relationship interactions follow the standard ActivityPub spec.
- `Follow`: request to receive statuses from a user, and view their statuses that have followers-only privacy
- `Accept`: approves a `Follow` and finalizes the relationship
- `Reject`: denies a `Follow`
- `Block`: prevent users from seeing one another's statuses, and prevents the blocked user from viewing the actor's profile
- `Update`: updates a user's profile and settings
- `Delete`: deactivates a user
- `Undo`: reverses a `Follow` or `Block`
### Activities
- `Create/Status`: saves a new status in the database.
- `Delete/Status`: Removes a status
- `Like/Status`: Creates a favorite on the status
- `Announce/Status`: Boosts the status into the actor's timeline
- `Undo/*`,: Reverses a `Like` or `Announce`
### Collections
User's books and lists are represented by [`OrderedCollection`](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection)
### Statuses
BookWyrm is focused on book reading activities - it is not a general-purpose messaging application. For this reason, BookWyrm only accepts status `Create` activities if they are:
- Direct messages (i.e., `Note`s with the privacy level `direct`, which mention a local user),
- Related to a book (of a custom status type that includes the field `inReplyToBook`),
- Replies to existing statuses saved in the database
All other statuses will be received by the instance inbox, but by design **will not be delivered to user inboxes or displayed to users**.
### Custom Object types
With the exception of `Note`, the following object types are used in Bookwyrm but are not currently provided with a custom JSON-LD `@context` extension IRI. This is likely to change in future to make them true deserialisable JSON-LD objects.
##### Note
Within BookWyrm a `Note` is constructed according to [the ActivityStreams vocabulary](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-note), however `Note`s can only be created as direct messages or as replies to other statuses. As mentioned above, this also applies to incoming `Note`s.
##### Review
A `Review` is a status in response to a book (indicated by the `inReplyToBook` field), which has a title, body, and numerical rating between 0 (not rated) and 5.
Example:
```json
{
"id": "https://example.net/user/library_lurker/review/2",
"type": "Review",
"published": "2023-06-30T21:43:46.013132+00:00",
"attributedTo": "https://example.net/user/library_lurker",
"content": "<p>This is an enjoyable book with great characters.</p>",
"to": ["https://example.net/user/library_lurker/followers"],
"cc": [],
"replies": {
"id": "https://example.net/user/library_lurker/review/2/replies",
"type": "OrderedCollection",
"totalItems": 0,
"first": "https://example.net/user/library_lurker/review/2/replies?page=1",
"last": "https://example.net/user/library_lurker/review/2/replies?page=1",
"@context": "https://www.w3.org/ns/activitystreams"
},
"summary": "Spoilers ahead!",
"tag": [],
"attachment": [],
"sensitive": true,
"inReplyToBook": "https://example.net/book/1",
"name": "What a cracking read",
"rating": 4.5,
"@context": "https://www.w3.org/ns/activitystreams"
}
```
##### Comment
A `Comment` on a book mentions a book and has a message body, reading status, and progress indicator.
Example:
```json
{
"id": "https://example.net/user/library_lurker/comment/9",
"type": "Comment",
"published": "2023-06-30T21:43:46.013132+00:00",
"attributedTo": "https://example.net/user/library_lurker",
"content": "<p>This is a very enjoyable book so far.</p>",
"to": ["https://example.net/user/library_lurker/followers"],
"cc": [],
"replies": {
"id": "https://example.net/user/library_lurker/comment/9/replies",
"type": "OrderedCollection",
"totalItems": 0,
"first": "https://example.net/user/library_lurker/comment/9/replies?page=1",
"last": "https://example.net/user/library_lurker/comment/9/replies?page=1",
"@context": "https://www.w3.org/ns/activitystreams"
},
"summary": "Spoilers ahead!",
"tag": [],
"attachment": [],
"sensitive": true,
"inReplyToBook": "https://example.net/book/1",
"readingStatus": "reading",
"progress": 25,
"progressMode": "PG",
"@context": "https://www.w3.org/ns/activitystreams"
}
```
##### Quotation
A quotation (aka "quote") has a message body, an excerpt from a book including position as a page number or percentage indicator, and mentions a book.
Example:
```json
{
"id": "https://example.net/user/mouse/quotation/13",
"url": "https://example.net/user/mouse/quotation/13",
"inReplyTo": null,
"published": "2020-05-10T02:38:31.150343+00:00",
"attributedTo": "https://example.net/user/mouse",
"to": [
"https://www.w3.org/ns/activitystreams#Public"
],
"cc": [
"https://example.net/user/mouse/followers"
],
"sensitive": false,
"content": "I really like this quote",
"type": "Quotation",
"replies": {
"id": "https://example.net/user/mouse/quotation/13/replies",
"type": "Collection",
"first": {
"type": "CollectionPage",
"next": "https://example.net/user/mouse/quotation/13/replies?only_other_accounts=true&page=true",
"partOf": "https://example.net/user/mouse/quotation/13/replies",
"items": []
}
},
"inReplyToBook": "https://example.net/book/1",
"quote": "To be or not to be, that is the question.",
"position": 50,
"positionMode": "PCT",
"@context": "https://www.w3.org/ns/activitystreams"
}
```
### Custom Objects
##### Work
A particular book, a "work" in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
Example:
```json
{
"id": "https://bookwyrm.social/book/5988",
"type": "Work",
"authors": [
"https://bookwyrm.social/author/417"
],
"first_published_date": null,
"published_date": null,
"title": "Piranesi",
"sort_title": null,
"subtitle": null,
"description": "**From the *New York Times* bestselling author of *Jonathan Strange & Mr. Norrell*, an intoxicating, hypnotic new novel set in a dreamlike alternative reality.",
"languages": [],
"series": null,
"series_number": null,
"subjects": [
"English literature"
],
"subject_places": [],
"openlibrary_key": "OL20893680W",
"librarything_key": null,
"goodreads_key": null,
"attachment": [
{
"url": "https://bookwyrm.social/images/covers/10226290-M.jpg",
"type": "Image"
}
],
"lccn": null,
"editions": [
"https://bookwyrm.social/book/5989"
],
"@context": "https://www.w3.org/ns/activitystreams"
}
```
##### Edition
A particular _manifestation_ of a Work, in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
Example:
```json
{
"id": "https://bookwyrm.social/book/5989",
"lastEditedBy": "https://example.net/users/rat",
"type": "Edition",
"authors": [
"https://bookwyrm.social/author/417"
],
"first_published_date": null,
"published_date": "2020-09-15T00:00:00+00:00",
"title": "Piranesi",
"sort_title": null,
"subtitle": null,
"description": "Piranesi's house is no ordinary building; its rooms are infinite, its corridors endless, its walls are lined with thousands upon thousands of statues, each one different from all the others.",
"languages": [
"English"
],
"series": null,
"series_number": null,
"subjects": [],
"subject_places": [],
"openlibrary_key": "OL29486417M",
"librarything_key": null,
"goodreads_key": null,
"isfdb": null,
"attachment": [
{
"url": "https://bookwyrm.social/images/covers/50202953._SX318_.jpg",
"type": "Image"
}
],
"isbn_10": "1526622424",
"isbn_13": "9781526622426",
"oclc_number": null,
"asin": null,
"pages": 272,
"physical_format": null,
"publishers": [
"Bloomsbury Publishing Plc"
],
"work": "https://bookwyrm.social/book/5988",
"@context": "https://www.w3.org/ns/activitystreams"
}
```
#### Shelf
A user's book collection. By default, every user has a `to-read`, `reading`, `read`, and `stopped-reading` shelf which are used to track reading progress. Users may create an unlimited number of additional shelves with their own ids.
Example
```json
{
"id": "https://example.net/user/avid_reader/books/extraspecialbooks-5",
"type": "Shelf",
"totalItems": 0,
"first": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
"last": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
"name": "Extra special books",
"owner": "https://example.net/user/avid_reader",
"to": [
"https://www.w3.org/ns/activitystreams#Public"
],
"cc": [
"https://example.net/user/avid_reader/followers"
],
"@context": "https://www.w3.org/ns/activitystreams"
}
```
#### List
A collection of books that may have items contributed by users other than the one who created the list.
Example:
```json
{
"id": "https://example.net/list/1",
"type": "BookList",
"totalItems": 0,
"first": "https://example.net/list/1?page=1",
"last": "https://example.net/list/1?page=1",
"name": "My cool list",
"owner": "https://example.net/user/avid_reader",
"to": [
"https://www.w3.org/ns/activitystreams#Public"
],
"cc": [
"https://example.net/user/avid_reader/followers"
],
"summary": "A list of books I like.",
"curation": "curated",
"@context": "https://www.w3.org/ns/activitystreams"
}
```
#### Activities
- `Create`: Adds a shelf or list to the database.
- `Delete`: Removes a shelf or list.
- `Add`: Adds a book to a shelf or list.
- `Remove`: Removes a book from a shelf or list.
## Alternative Serialization
Because BookWyrm uses custom object types that aren't listed in [the standard ActivityStreams Vocabulary](https://www.w3.org/TR/activitystreams-vocabulary), some statuses are transformed into standard types when sent to or viewed by non-BookWyrm services. `Review`s are converted into `Article`s, and `Comment`s and `Quotation`s are converted into `Note`s, with a link to the book and the cover image attached.
In future this may be done with [JSON-LD type arrays](https://www.w3.org/TR/json-ld/#specifying-the-type) instead.
## Other extensions
### Webfinger
Bookwyrm uses the [Webfinger](https://datatracker.ietf.org/doc/html/rfc7033) standard to identify and disambiguate fediverse actors. The [Webfinger documentation on the Mastodon project](https://docs.joinmastodon.org/spec/webfinger/) provides a good overview of how Webfinger is used.
### HTTP Signatures
Bookwyrm uses and requires HTTP signatures for all `POST` requests. `GET` requests are not signed by default, but if Bookwyrm receives a `403` response to a `GET` it will re-send the request, signed by the default server user. This usually will have a user id of `https://example.net/user/bookwyrm.instance.actor`
#### publicKey id
In older versions of Bookwyrm the `publicKey.id` was incorrectly listed in request headers as `https://example.net/user/username#main-key`. As of v0.6.3 the id is now listed correctly, as `https://example.net/user/username/#main-key`. In most ActivityPub implementations this will make no difference as the URL will usually resolve to the same place.
### NodeInfo
Bookwyrm uses the [NodeInfo](http://nodeinfo.diaspora.software/) standard to provide statistics and version information for each instance.
## Further Documentation
See [docs.joinbookwyrm.com/](https://docs.joinbookwyrm.com/) for more documentation.

View file

@ -37,7 +37,7 @@ Keep track of what books you've read, and what books you'd like to read in the f
Federation allows you to interact with users on other instances and services, and also shares metadata about books and authors, which collaboratively builds a decentralized database of books. Federation allows you to interact with users on other instances and services, and also shares metadata about books and authors, which collaboratively builds a decentralized database of books.
### Privacy and moderation ### Privacy and moderation
Users and administrators can control who can see thier posts and what other instances to federate with. Users and administrators can control who can see their posts and what other instances to federate with.
## Tech Stack ## Tech Stack
Web backend Web backend

View file

@ -3,8 +3,12 @@ import inspect
import sys import sys
from .base_activity import ActivityEncoder, Signature, naive_parse from .base_activity import ActivityEncoder, Signature, naive_parse
from .base_activity import Link, Mention from .base_activity import Link, Mention, Hashtag
from .base_activity import ActivitySerializerError, resolve_remote_id from .base_activity import (
ActivitySerializerError,
resolve_remote_id,
get_representative,
)
from .image import Document, Image from .image import Document, Image
from .note import Note, GeneratedNote, Article, Comment, Quotation from .note import Note, GeneratedNote, Article, Comment, Quotation
from .note import Review, Rating from .note import Review, Rating

View file

@ -1,16 +1,27 @@
""" basics for an activitypub serializer """ """ basics for an activitypub serializer """
from __future__ import annotations
from dataclasses import dataclass, fields, MISSING from dataclasses import dataclass, fields, MISSING
from json import JSONEncoder from json import JSONEncoder
import logging import logging
from typing import Optional, Union, TypeVar, overload, Any
import requests
from django.apps import apps from django.apps import apps
from django.db import IntegrityError, transaction from django.db import IntegrityError, transaction
from django.utils.http import http_date
from bookwyrm import models
from bookwyrm.connectors import ConnectorException, get_data from bookwyrm.connectors import ConnectorException, get_data
from bookwyrm.tasks import app, MEDIUM from bookwyrm.models import base_model
from bookwyrm.signatures import make_signature
from bookwyrm.settings import DOMAIN, INSTANCE_ACTOR_USERNAME
from bookwyrm.tasks import app, MISC
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel)
class ActivitySerializerError(ValueError): class ActivitySerializerError(ValueError):
"""routine problems serializing activitypub json""" """routine problems serializing activitypub json"""
@ -60,7 +71,13 @@ class ActivityObject:
id: str id: str
type: str type: str
def __init__(self, activity_objects=None, **kwargs): def __init__(
self,
activity_objects: Optional[
dict[str, Union[str, list[str], ActivityObject, base_model.BookWyrmModel]]
] = None,
**kwargs: Any,
):
"""this lets you pass in an object with fields that aren't in the """this lets you pass in an object with fields that aren't in the
dataclass, which it ignores. Any field in the dataclass is required or dataclass, which it ignores. Any field in the dataclass is required or
has a default value""" has a default value"""
@ -95,16 +112,34 @@ class ActivityObject:
# pylint: disable=too-many-locals,too-many-branches,too-many-arguments # pylint: disable=too-many-locals,too-many-branches,too-many-arguments
def to_model( def to_model(
self, model=None, instance=None, allow_create=True, save=True, overwrite=True self,
): model: Optional[type[TBookWyrmModel]] = None,
"""convert from an activity to a model instance""" instance: Optional[TBookWyrmModel] = None,
allow_create: bool = True,
save: bool = True,
overwrite: bool = True,
allow_external_connections: bool = True,
) -> Optional[TBookWyrmModel]:
"""convert from an activity to a model instance. Args:
model: the django model that this object is being converted to
(will guess if not known)
instance: an existing database entry that is going to be updated by
this activity
allow_create: whether a new object should be created if there is no
existing object is provided or found matching the remote_id
save: store in the database if true, return an unsaved model obj if false
overwrite: replace fields in the database with this activity if true,
only update blank fields if false
allow_external_connections: look up missing data if true,
throw an exception if false and an external connection is needed
"""
model = model or get_model_from_type(self.type) model = model or get_model_from_type(self.type)
# only reject statuses if we're potentially creating them # only reject statuses if we're potentially creating them
if ( if (
allow_create allow_create
and hasattr(model, "ignore_activity") and hasattr(model, "ignore_activity")
and model.ignore_activity(self) and model.ignore_activity(self, allow_external_connections)
): ):
return None return None
@ -122,7 +157,10 @@ class ActivityObject:
for field in instance.simple_fields: for field in instance.simple_fields:
try: try:
changed = field.set_field_from_activity( changed = field.set_field_from_activity(
instance, self, overwrite=overwrite instance,
self,
overwrite=overwrite,
allow_external_connections=allow_external_connections,
) )
if changed: if changed:
update_fields.append(field.name) update_fields.append(field.name)
@ -133,7 +171,11 @@ class ActivityObject:
# too early and jank up users # too early and jank up users
for field in instance.image_fields: for field in instance.image_fields:
changed = field.set_field_from_activity( changed = field.set_field_from_activity(
instance, self, save=save, overwrite=overwrite instance,
self,
save=save,
overwrite=overwrite,
allow_external_connections=allow_external_connections,
) )
if changed: if changed:
update_fields.append(field.name) update_fields.append(field.name)
@ -156,8 +198,12 @@ class ActivityObject:
# add many to many fields, which have to be set post-save # add many to many fields, which have to be set post-save
for field in instance.many_to_many_fields: for field in instance.many_to_many_fields:
# mention books/users, for example # mention books/users/hashtags, for example
field.set_field_from_activity(instance, self) field.set_field_from_activity(
instance,
self,
allow_external_connections=allow_external_connections,
)
# reversed relationships in the models # reversed relationships in the models
for ( for (
@ -194,6 +240,11 @@ class ActivityObject:
try: try:
if issubclass(type(v), ActivityObject): if issubclass(type(v), ActivityObject):
data[k] = v.serialize() data[k] = v.serialize()
elif isinstance(v, list):
data[k] = [
e.serialize() if issubclass(type(e), ActivityObject) else e
for e in v
]
except TypeError: except TypeError:
pass pass
data = {k: v for (k, v) in data.items() if v is not None and k not in omit} data = {k: v for (k, v) in data.items() if v is not None and k not in omit}
@ -202,7 +253,7 @@ class ActivityObject:
return data return data
@app.task(queue=MEDIUM) @app.task(queue=MISC)
@transaction.atomic @transaction.atomic
def set_related_field( def set_related_field(
model_name, origin_model_name, related_field_name, related_remote_id, data model_name, origin_model_name, related_field_name, related_remote_id, data
@ -241,10 +292,10 @@ def set_related_field(
def get_model_from_type(activity_type): def get_model_from_type(activity_type):
"""given the activity, what type of model""" """given the activity, what type of model"""
models = apps.get_models() activity_models = apps.get_models()
model = [ model = [
m m
for m in models for m in activity_models
if hasattr(m, "activity_serializer") if hasattr(m, "activity_serializer")
and hasattr(m.activity_serializer, "type") and hasattr(m.activity_serializer, "type")
and m.activity_serializer.type == activity_type and m.activity_serializer.type == activity_type
@ -256,10 +307,48 @@ def get_model_from_type(activity_type):
return model[0] return model[0]
# pylint: disable=too-many-arguments
@overload
def resolve_remote_id( def resolve_remote_id(
remote_id, model=None, refresh=False, save=True, get_activity=False remote_id: str,
): model: type[TBookWyrmModel],
"""take a remote_id and return an instance, creating if necessary""" refresh: bool = False,
save: bool = True,
get_activity: bool = False,
allow_external_connections: bool = True,
) -> TBookWyrmModel:
...
# pylint: disable=too-many-arguments
@overload
def resolve_remote_id(
remote_id: str,
model: Optional[str] = None,
refresh: bool = False,
save: bool = True,
get_activity: bool = False,
allow_external_connections: bool = True,
) -> base_model.BookWyrmModel:
...
# pylint: disable=too-many-arguments
def resolve_remote_id(
remote_id: str,
model: Optional[Union[str, type[base_model.BookWyrmModel]]] = None,
refresh: bool = False,
save: bool = True,
get_activity: bool = False,
allow_external_connections: bool = True,
) -> base_model.BookWyrmModel:
"""take a remote_id and return an instance, creating if necessary. Args:
remote_id: the unique url for looking up the object in the db or by http
model: a string or object representing the model that corresponds to the object
save: whether to return an unsaved database entry or a saved one
get_activity: whether to return the activitypub object or the model object
allow_external_connections: whether to make http connections
"""
if model: # a bonus check we can do if we already know the model if model: # a bonus check we can do if we already know the model
if isinstance(model, str): if isinstance(model, str):
model = apps.get_model(f"bookwyrm.{model}", require_ready=True) model = apps.get_model(f"bookwyrm.{model}", require_ready=True)
@ -267,13 +356,26 @@ def resolve_remote_id(
if result and not refresh: if result and not refresh:
return result if not get_activity else result.to_activity_dataclass() return result if not get_activity else result.to_activity_dataclass()
# The above block will return the object if it already exists in the database.
# If it doesn't, an external connection would be needed, so check if that's cool
if not allow_external_connections:
raise ActivitySerializerError(
"Unable to serialize object without making external HTTP requests"
)
# load the data and create the object # load the data and create the object
try: try:
data = get_data(remote_id) data = get_data(remote_id)
except ConnectorException: except ConnectionError:
logger.exception("Could not connect to host for remote_id: %s", remote_id) logger.info("Could not connect to host for remote_id: %s", remote_id)
return None return None
except requests.HTTPError as e:
if (e.response is not None) and e.response.status_code == 401:
# This most likely means it's a mastodon with secure fetch enabled.
data = get_activitypub_data(remote_id)
else:
logger.info("Could not connect to host for remote_id: %s", remote_id)
return None
# determine the model implicitly, if not provided # determine the model implicitly, if not provided
# or if it's a model with subclasses like Status, check again # or if it's a model with subclasses like Status, check again
if not model or hasattr(model.objects, "select_subclasses"): if not model or hasattr(model.objects, "select_subclasses"):
@ -292,6 +394,52 @@ def resolve_remote_id(
return item.to_model(model=model, instance=result, save=save) return item.to_model(model=model, instance=result, save=save)
def get_representative():
"""Get or create an actor representing the instance
to sign requests to 'secure mastodon' servers"""
username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
email = "bookwyrm@localhost"
try:
user = models.User.objects.get(username=username)
except models.User.DoesNotExist:
user = models.User.objects.create_user(
username=username,
email=email,
local=True,
localname=INSTANCE_ACTOR_USERNAME,
)
return user
def get_activitypub_data(url):
"""wrapper for request.get"""
now = http_date()
sender = get_representative()
if not sender.key_pair.private_key:
# this shouldn't happen. it would be bad if it happened.
raise ValueError("No private key found for sender")
try:
resp = requests.get(
url,
headers={
# pylint: disable=line-too-long
"Accept": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
"Date": now,
"Signature": make_signature("get", sender, url, now),
},
)
except requests.RequestException:
raise ConnectorException()
if not resp.ok:
resp.raise_for_status()
try:
data = resp.json()
except ValueError:
raise ConnectorException()
return data
@dataclass(init=False) @dataclass(init=False)
class Link(ActivityObject): class Link(ActivityObject):
"""for tagging a book in a status""" """for tagging a book in a status"""
@ -306,7 +454,9 @@ class Link(ActivityObject):
def serialize(self, **kwargs): def serialize(self, **kwargs):
"""remove fields""" """remove fields"""
omit = ("id", "type", "@context") omit = ("id", "@context")
if self.type == "Link":
omit += ("type",)
return super().serialize(omit=omit) return super().serialize(omit=omit)
@ -315,3 +465,10 @@ class Mention(Link):
"""a subtype of Link for mentioning an actor""" """a subtype of Link for mentioning an actor"""
type: str = "Mention" type: str = "Mention"
@dataclass(init=False)
class Hashtag(Link):
"""a subtype of Link for mentioning a hashtag"""
type: str = "Hashtag"

View file

@ -1,6 +1,6 @@
""" book and author data """ """ book and author data """
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import List from typing import Optional
from .base_activity import ActivityObject from .base_activity import ActivityObject
from .image import Document from .image import Document
@ -11,17 +11,19 @@ from .image import Document
class BookData(ActivityObject): class BookData(ActivityObject):
"""shared fields for all book data and authors""" """shared fields for all book data and authors"""
openlibraryKey: str = None openlibraryKey: Optional[str] = None
inventaireId: str = None inventaireId: Optional[str] = None
librarythingKey: str = None librarythingKey: Optional[str] = None
goodreadsKey: str = None goodreadsKey: Optional[str] = None
bnfId: str = None bnfId: Optional[str] = None
viaf: str = None viaf: Optional[str] = None
wikidata: str = None wikidata: Optional[str] = None
asin: str = None asin: Optional[str] = None
lastEditedBy: str = None aasin: Optional[str] = None
links: List[str] = field(default_factory=lambda: []) isfdb: Optional[str] = None
fileLinks: List[str] = field(default_factory=lambda: []) lastEditedBy: Optional[str] = None
links: list[str] = field(default_factory=list)
fileLinks: list[str] = field(default_factory=list)
# pylint: disable=invalid-name # pylint: disable=invalid-name
@ -33,17 +35,17 @@ class Book(BookData):
sortTitle: str = None sortTitle: str = None
subtitle: str = None subtitle: str = None
description: str = "" description: str = ""
languages: List[str] = field(default_factory=lambda: []) languages: list[str] = field(default_factory=list)
series: str = "" series: str = ""
seriesNumber: str = "" seriesNumber: str = ""
subjects: List[str] = field(default_factory=lambda: []) subjects: list[str] = field(default_factory=list)
subjectPlaces: List[str] = field(default_factory=lambda: []) subjectPlaces: list[str] = field(default_factory=list)
authors: List[str] = field(default_factory=lambda: []) authors: list[str] = field(default_factory=list)
firstPublishedDate: str = "" firstPublishedDate: str = ""
publishedDate: str = "" publishedDate: str = ""
cover: Document = None cover: Optional[Document] = None
type: str = "Book" type: str = "Book"
@ -56,10 +58,10 @@ class Edition(Book):
isbn10: str = "" isbn10: str = ""
isbn13: str = "" isbn13: str = ""
oclcNumber: str = "" oclcNumber: str = ""
pages: int = None pages: Optional[int] = None
physicalFormat: str = "" physicalFormat: str = ""
physicalFormatDetail: str = "" physicalFormatDetail: str = ""
publishers: List[str] = field(default_factory=lambda: []) publishers: list[str] = field(default_factory=list)
editionRank: int = 0 editionRank: int = 0
type: str = "Edition" type: str = "Edition"
@ -71,7 +73,7 @@ class Work(Book):
"""work instance of a book object""" """work instance of a book object"""
lccn: str = "" lccn: str = ""
editions: List[str] = field(default_factory=lambda: []) editions: list[str] = field(default_factory=list)
type: str = "Work" type: str = "Work"
@ -81,12 +83,13 @@ class Author(BookData):
"""author of a book""" """author of a book"""
name: str name: str
isni: str = None isni: Optional[str] = None
viafId: str = None viafId: Optional[str] = None
gutenbergId: str = None gutenbergId: Optional[str] = None
born: str = None born: Optional[str] = None
died: str = None died: Optional[str] = None
aliases: List[str] = field(default_factory=lambda: []) aliases: list[str] = field(default_factory=list)
bio: str = "" bio: str = ""
wikipediaLink: str = "" wikipediaLink: str = ""
type: str = "Author" type: str = "Author"
website: str = ""

View file

@ -1,9 +1,12 @@
""" note serializer and children thereof """ """ note serializer and children thereof """
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import Dict, List from typing import Dict, List
from django.apps import apps import re
from .base_activity import ActivityObject, Link from django.apps import apps
from django.db import IntegrityError, transaction
from .base_activity import ActivityObject, ActivitySerializerError, Link
from .image import Document from .image import Document
@ -38,6 +41,47 @@ class Note(ActivityObject):
updated: str = None updated: str = None
type: str = "Note" type: str = "Note"
# pylint: disable=too-many-arguments
def to_model(
self,
model=None,
instance=None,
allow_create=True,
save=True,
overwrite=True,
allow_external_connections=True,
):
instance = super().to_model(
model, instance, allow_create, save, overwrite, allow_external_connections
)
if instance is None:
return instance
# Replace links to hashtags in content with local URLs
changed_content = False
for hashtag in instance.mention_hashtags.all():
updated_content = re.sub(
rf'(<a href=")[^"]*(" data-mention="hashtag">{hashtag.name}</a>)',
rf"\1{hashtag.remote_id}\2",
instance.content,
flags=re.IGNORECASE,
)
if instance.content != updated_content:
instance.content = updated_content
changed_content = True
if not save or not changed_content:
return instance
with transaction.atomic():
try:
instance.save(broadcast=False, update_fields=["content"])
except IntegrityError as e:
raise ActivitySerializerError(e)
return instance
@dataclass(init=False) @dataclass(init=False)
class Article(Note): class Article(Note):

View file

@ -14,12 +14,12 @@ class Verb(ActivityObject):
actor: str actor: str
object: ActivityObject object: ActivityObject
def action(self): def action(self, allow_external_connections=True):
"""usually we just want to update and save""" """usually we just want to update and save"""
# self.object may return None if the object is invalid in an expected way # self.object may return None if the object is invalid in an expected way
# ie, Question type # ie, Question type
if self.object: if self.object:
self.object.to_model() self.object.to_model(allow_external_connections=allow_external_connections)
# pylint: disable=invalid-name # pylint: disable=invalid-name
@ -42,7 +42,7 @@ class Delete(Verb):
cc: List[str] = field(default_factory=lambda: []) cc: List[str] = field(default_factory=lambda: [])
type: str = "Delete" type: str = "Delete"
def action(self): def action(self, allow_external_connections=True):
"""find and delete the activity object""" """find and delete the activity object"""
if not self.object: if not self.object:
return return
@ -52,7 +52,11 @@ class Delete(Verb):
model = apps.get_model("bookwyrm.User") model = apps.get_model("bookwyrm.User")
obj = model.find_existing_by_remote_id(self.object) obj = model.find_existing_by_remote_id(self.object)
else: else:
obj = self.object.to_model(save=False, allow_create=False) obj = self.object.to_model(
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
)
if obj: if obj:
obj.delete() obj.delete()
@ -67,11 +71,13 @@ class Update(Verb):
to: List[str] to: List[str]
type: str = "Update" type: str = "Update"
def action(self): def action(self, allow_external_connections=True):
"""update a model instance from the dataclass""" """update a model instance from the dataclass"""
if not self.object: if not self.object:
return return
self.object.to_model(allow_create=False) self.object.to_model(
allow_create=False, allow_external_connections=allow_external_connections
)
@dataclass(init=False) @dataclass(init=False)
@ -80,10 +86,10 @@ class Undo(Verb):
type: str = "Undo" type: str = "Undo"
def action(self): def action(self, allow_external_connections=True):
"""find and remove the activity object""" """find and remove the activity object"""
if isinstance(self.object, str): if isinstance(self.object, str):
# it may be that sometihng should be done with these, but idk what # it may be that something should be done with these, but idk what
# this seems just to be coming from pleroma # this seems just to be coming from pleroma
return return
@ -92,13 +98,28 @@ class Undo(Verb):
model = None model = None
if self.object.type == "Follow": if self.object.type == "Follow":
model = apps.get_model("bookwyrm.UserFollows") model = apps.get_model("bookwyrm.UserFollows")
obj = self.object.to_model(model=model, save=False, allow_create=False) obj = self.object.to_model(
model=model,
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
)
if not obj: if not obj:
# this could be a folloq request not a follow proper # this could be a follow request not a follow proper
model = apps.get_model("bookwyrm.UserFollowRequest") model = apps.get_model("bookwyrm.UserFollowRequest")
obj = self.object.to_model(model=model, save=False, allow_create=False) obj = self.object.to_model(
model=model,
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
)
else: else:
obj = self.object.to_model(model=model, save=False, allow_create=False) obj = self.object.to_model(
model=model,
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
)
if not obj: if not obj:
# if we don't have the object, we can't undo it. happens a lot with boosts # if we don't have the object, we can't undo it. happens a lot with boosts
return return
@ -112,9 +133,9 @@ class Follow(Verb):
object: str object: str
type: str = "Follow" type: str = "Follow"
def action(self): def action(self, allow_external_connections=True):
"""relationship save""" """relationship save"""
self.to_model() self.to_model(allow_external_connections=allow_external_connections)
@dataclass(init=False) @dataclass(init=False)
@ -124,9 +145,9 @@ class Block(Verb):
object: str object: str
type: str = "Block" type: str = "Block"
def action(self): def action(self, allow_external_connections=True):
"""relationship save""" """relationship save"""
self.to_model() self.to_model(allow_external_connections=allow_external_connections)
@dataclass(init=False) @dataclass(init=False)
@ -136,7 +157,7 @@ class Accept(Verb):
object: Follow object: Follow
type: str = "Accept" type: str = "Accept"
def action(self): def action(self, allow_external_connections=True):
"""accept a request""" """accept a request"""
obj = self.object.to_model(save=False, allow_create=True) obj = self.object.to_model(save=False, allow_create=True)
obj.accept() obj.accept()
@ -149,7 +170,7 @@ class Reject(Verb):
object: Follow object: Follow
type: str = "Reject" type: str = "Reject"
def action(self): def action(self, allow_external_connections=True):
"""reject a follow request""" """reject a follow request"""
obj = self.object.to_model(save=False, allow_create=False) obj = self.object.to_model(save=False, allow_create=False)
obj.reject() obj.reject()
@ -163,7 +184,7 @@ class Add(Verb):
object: CollectionItem object: CollectionItem
type: str = "Add" type: str = "Add"
def action(self): def action(self, allow_external_connections=True):
"""figure out the target to assign the item to a collection""" """figure out the target to assign the item to a collection"""
target = resolve_remote_id(self.target) target = resolve_remote_id(self.target)
item = self.object.to_model(save=False) item = self.object.to_model(save=False)
@ -177,7 +198,7 @@ class Remove(Add):
type: str = "Remove" type: str = "Remove"
def action(self): def action(self, allow_external_connections=True):
"""find and remove the activity object""" """find and remove the activity object"""
obj = self.object.to_model(save=False, allow_create=False) obj = self.object.to_model(save=False, allow_create=False)
if obj: if obj:
@ -191,9 +212,9 @@ class Like(Verb):
object: str object: str
type: str = "Like" type: str = "Like"
def action(self): def action(self, allow_external_connections=True):
"""like""" """like"""
self.to_model() self.to_model(allow_external_connections=allow_external_connections)
# pylint: disable=invalid-name # pylint: disable=invalid-name
@ -207,6 +228,6 @@ class Announce(Verb):
object: str object: str
type: str = "Announce" type: str = "Announce"
def action(self): def action(self, allow_external_connections=True):
"""boost""" """boost"""
self.to_model() self.to_model(allow_external_connections=allow_external_connections)

View file

@ -4,27 +4,32 @@ from django.dispatch import receiver
from django.db import transaction from django.db import transaction
from django.db.models import signals, Q from django.db.models import signals, Q
from django.utils import timezone from django.utils import timezone
from opentelemetry import trace
from bookwyrm import models from bookwyrm import models
from bookwyrm.redis_store import RedisStore, r from bookwyrm.redis_store import RedisStore, r
from bookwyrm.tasks import app, LOW, MEDIUM, HIGH from bookwyrm.tasks import app, STREAMS, IMPORT_TRIGGERED
from bookwyrm.telemetry import open_telemetry
tracer = open_telemetry.tracer()
class ActivityStream(RedisStore): class ActivityStream(RedisStore):
"""a category of activity stream (like home, local, books)""" """a category of activity stream (like home, local, books)"""
def stream_id(self, user): def stream_id(self, user_id):
"""the redis key for this user's instance of this stream""" """the redis key for this user's instance of this stream"""
return f"{user.id}-{self.key}" return f"{user_id}-{self.key}"
def unread_id(self, user): def unread_id(self, user_id):
"""the redis key for this user's unread count for this stream""" """the redis key for this user's unread count for this stream"""
stream_id = self.stream_id(user) stream_id = self.stream_id(user_id)
return f"{stream_id}-unread" return f"{stream_id}-unread"
def unread_by_status_type_id(self, user): def unread_by_status_type_id(self, user_id):
"""the redis key for this user's unread count for this stream""" """the redis key for this user's unread count for this stream"""
stream_id = self.stream_id(user) stream_id = self.stream_id(user_id)
return f"{stream_id}-unread-by-type" return f"{stream_id}-unread-by-type"
def get_rank(self, obj): # pylint: disable=no-self-use def get_rank(self, obj): # pylint: disable=no-self-use
@ -33,16 +38,19 @@ class ActivityStream(RedisStore):
def add_status(self, status, increment_unread=False): def add_status(self, status, increment_unread=False):
"""add a status to users' feeds""" """add a status to users' feeds"""
audience = self.get_audience(status)
# the pipeline contains all the add-to-stream activities # the pipeline contains all the add-to-stream activities
pipeline = self.add_object_to_related_stores(status, execute=False) pipeline = self.add_object_to_stores(
status, self.get_stores_for_users(audience), execute=False
)
if increment_unread: if increment_unread:
for user in self.get_audience(status): for user_id in audience:
# add to the unread status count # add to the unread status count
pipeline.incr(self.unread_id(user)) pipeline.incr(self.unread_id(user_id))
# add to the unread status count for status type # add to the unread status count for status type
pipeline.hincrby( pipeline.hincrby(
self.unread_by_status_type_id(user), get_status_type(status), 1 self.unread_by_status_type_id(user_id), get_status_type(status), 1
) )
# and go! # and go!
@ -52,21 +60,21 @@ class ActivityStream(RedisStore):
"""add a user's statuses to another user's feed""" """add a user's statuses to another user's feed"""
# only add the statuses that the viewer should be able to see (ie, not dms) # only add the statuses that the viewer should be able to see (ie, not dms)
statuses = models.Status.privacy_filter(viewer).filter(user=user) statuses = models.Status.privacy_filter(viewer).filter(user=user)
self.bulk_add_objects_to_store(statuses, self.stream_id(viewer)) self.bulk_add_objects_to_store(statuses, self.stream_id(viewer.id))
def remove_user_statuses(self, viewer, user): def remove_user_statuses(self, viewer, user):
"""remove a user's status from another user's feed""" """remove a user's status from another user's feed"""
# remove all so that followers only statuses are removed # remove all so that followers only statuses are removed
statuses = user.status_set.all() statuses = user.status_set.all()
self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer)) self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer.id))
def get_activity_stream(self, user): def get_activity_stream(self, user):
"""load the statuses to be displayed""" """load the statuses to be displayed"""
# clear unreads for this feed # clear unreads for this feed
r.set(self.unread_id(user), 0) r.set(self.unread_id(user.id), 0)
r.delete(self.unread_by_status_type_id(user)) r.delete(self.unread_by_status_type_id(user.id))
statuses = self.get_store(self.stream_id(user)) statuses = self.get_store(self.stream_id(user.id))
return ( return (
models.Status.objects.select_subclasses() models.Status.objects.select_subclasses()
.filter(id__in=statuses) .filter(id__in=statuses)
@ -83,11 +91,11 @@ class ActivityStream(RedisStore):
def get_unread_count(self, user): def get_unread_count(self, user):
"""get the unread status count for this user's feed""" """get the unread status count for this user's feed"""
return int(r.get(self.unread_id(user)) or 0) return int(r.get(self.unread_id(user.id)) or 0)
def get_unread_count_by_status_type(self, user): def get_unread_count_by_status_type(self, user):
"""get the unread status count for this user's feed's status types""" """get the unread status count for this user's feed's status types"""
status_types = r.hgetall(self.unread_by_status_type_id(user)) status_types = r.hgetall(self.unread_by_status_type_id(user.id))
return { return {
str(key.decode("utf-8")): int(value) or 0 str(key.decode("utf-8")): int(value) or 0
for key, value in status_types.items() for key, value in status_types.items()
@ -95,13 +103,20 @@ class ActivityStream(RedisStore):
def populate_streams(self, user): def populate_streams(self, user):
"""go from zero to a timeline""" """go from zero to a timeline"""
self.populate_store(self.stream_id(user)) self.populate_store(self.stream_id(user.id))
def get_audience(self, status): # pylint: disable=no-self-use @tracer.start_as_current_span("ActivityStream._get_audience")
"""given a status, what users should see it""" def _get_audience(self, status): # pylint: disable=no-self-use
# direct messages don't appeard in feeds, direct comments/reviews/etc do """given a status, what users should see it, excluding the author"""
trace.get_current_span().set_attribute("status_type", status.status_type)
trace.get_current_span().set_attribute("status_privacy", status.privacy)
trace.get_current_span().set_attribute(
"status_reply_parent_privacy",
status.reply_parent.privacy if status.reply_parent else status.privacy,
)
# direct messages don't appear in feeds, direct comments/reviews/etc do
if status.privacy == "direct" and status.status_type == "Note": if status.privacy == "direct" and status.status_type == "Note":
return [] return models.User.objects.none()
# everybody who could plausibly see this status # everybody who could plausibly see this status
audience = models.User.objects.filter( audience = models.User.objects.filter(
@ -114,15 +129,13 @@ class ActivityStream(RedisStore):
# only visible to the poster and mentioned users # only visible to the poster and mentioned users
if status.privacy == "direct": if status.privacy == "direct":
audience = audience.filter( audience = audience.filter(
Q(id=status.user.id) # if the user is the post's author Q(id__in=status.mention_users.all()) # if the user is mentioned
| Q(id__in=status.mention_users.all()) # if the user is mentioned
) )
# don't show replies to statuses the user can't see # don't show replies to statuses the user can't see
elif status.reply_parent and status.reply_parent.privacy == "followers": elif status.reply_parent and status.reply_parent.privacy == "followers":
audience = audience.filter( audience = audience.filter(
Q(id=status.user.id) # if the user is the post's author Q(id=status.reply_parent.user.id) # if the user is the OG author
| Q(id=status.reply_parent.user.id) # if the user is the OG author
| ( | (
Q(following=status.user) & Q(following=status.reply_parent.user) Q(following=status.user) & Q(following=status.reply_parent.user)
) # if the user is following both authors ) # if the user is following both authors
@ -131,13 +144,23 @@ class ActivityStream(RedisStore):
# only visible to the poster's followers and tagged users # only visible to the poster's followers and tagged users
elif status.privacy == "followers": elif status.privacy == "followers":
audience = audience.filter( audience = audience.filter(
Q(id=status.user.id) # if the user is the post's author Q(following=status.user) # if the user is following the author
| Q(following=status.user) # if the user is following the author
) )
return audience.distinct() return audience.distinct()
def get_stores_for_object(self, obj): @tracer.start_as_current_span("ActivityStream.get_audience")
return [self.stream_id(u) for u in self.get_audience(obj)] def get_audience(self, status):
"""given a status, what users should see it"""
trace.get_current_span().set_attribute("stream_id", self.key)
audience = self._get_audience(status).values_list("id", flat=True)
status_author = models.User.objects.filter(
is_active=True, local=True, id=status.user.id
).values_list("id", flat=True)
return list(set(list(audience) + list(status_author)))
def get_stores_for_users(self, user_ids):
"""convert a list of user ids into redis store ids"""
return [self.stream_id(user_id) for user_id in user_ids]
def get_statuses_for_user(self, user): # pylint: disable=no-self-use def get_statuses_for_user(self, user): # pylint: disable=no-self-use
"""given a user, what statuses should they see on this stream""" """given a user, what statuses should they see on this stream"""
@ -156,14 +179,19 @@ class HomeStream(ActivityStream):
key = "home" key = "home"
@tracer.start_as_current_span("HomeStream.get_audience")
def get_audience(self, status): def get_audience(self, status):
audience = super().get_audience(status) trace.get_current_span().set_attribute("stream_id", self.key)
audience = super()._get_audience(status)
if not audience: if not audience:
return [] return []
return audience.filter( # if the user is following the author
Q(id=status.user.id) # if the user is the post's author audience = audience.filter(following=status.user).values_list("id", flat=True)
| Q(following=status.user) # if the user is following the author # if the user is the post's author
).distinct() status_author = models.User.objects.filter(
is_active=True, local=True, id=status.user.id
).values_list("id", flat=True)
return list(set(list(audience) + list(status_author)))
def get_statuses_for_user(self, user): def get_statuses_for_user(self, user):
return models.Status.privacy_filter( return models.Status.privacy_filter(
@ -202,8 +230,20 @@ class BooksStream(ActivityStream):
key = "books" key = "books"
def get_audience(self, status): def _get_audience(self, status):
"""anyone with the mentioned book on their shelves""" """anyone with the mentioned book on their shelves"""
work = (
status.book.parent_work
if hasattr(status, "book")
else status.mention_books.first().parent_work
)
audience = super()._get_audience(status)
if not audience:
return models.User.objects.none()
return audience.filter(shelfbook__book__parent_work=work).distinct()
def get_audience(self, status):
# only show public statuses on the books feed, # only show public statuses on the books feed,
# and only statuses that mention books # and only statuses that mention books
if status.privacy != "public" or not ( if status.privacy != "public" or not (
@ -211,16 +251,7 @@ class BooksStream(ActivityStream):
): ):
return [] return []
work = ( return super().get_audience(status)
status.book.parent_work
if hasattr(status, "book")
else status.mention_books.first().parent_work
)
audience = super().get_audience(status)
if not audience:
return []
return audience.filter(shelfbook__book__parent_work=work).distinct()
def get_statuses_for_user(self, user): def get_statuses_for_user(self, user):
"""any public status that mentions the user's books""" """any public status that mentions the user's books"""
@ -244,38 +275,38 @@ class BooksStream(ActivityStream):
def add_book_statuses(self, user, book): def add_book_statuses(self, user, book):
"""add statuses about a book to a user's feed""" """add statuses about a book to a user's feed"""
work = book.parent_work work = book.parent_work
statuses = ( statuses = models.Status.privacy_filter(
models.Status.privacy_filter( user,
user, privacy_levels=["public"],
privacy_levels=["public"],
)
.filter(
Q(comment__book__parent_work=work)
| Q(quotation__book__parent_work=work)
| Q(review__book__parent_work=work)
| Q(mention_books__parent_work=work)
)
.distinct()
) )
self.bulk_add_objects_to_store(statuses, self.stream_id(user))
book_comments = statuses.filter(Q(comment__book__parent_work=work))
book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
book_reviews = statuses.filter(Q(review__book__parent_work=work))
book_mentions = statuses.filter(Q(mention_books__parent_work=work))
self.bulk_add_objects_to_store(book_comments, self.stream_id(user.id))
self.bulk_add_objects_to_store(book_quotations, self.stream_id(user.id))
self.bulk_add_objects_to_store(book_reviews, self.stream_id(user.id))
self.bulk_add_objects_to_store(book_mentions, self.stream_id(user.id))
def remove_book_statuses(self, user, book): def remove_book_statuses(self, user, book):
"""add statuses about a book to a user's feed""" """add statuses about a book to a user's feed"""
work = book.parent_work work = book.parent_work
statuses = ( statuses = models.Status.privacy_filter(
models.Status.privacy_filter( user,
user, privacy_levels=["public"],
privacy_levels=["public"],
)
.filter(
Q(comment__book__parent_work=work)
| Q(quotation__book__parent_work=work)
| Q(review__book__parent_work=work)
| Q(mention_books__parent_work=work)
)
.distinct()
) )
self.bulk_remove_objects_from_store(statuses, self.stream_id(user))
book_comments = statuses.filter(Q(comment__book__parent_work=work))
book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
book_reviews = statuses.filter(Q(review__book__parent_work=work))
book_mentions = statuses.filter(Q(mention_books__parent_work=work))
self.bulk_remove_objects_from_store(book_comments, self.stream_id(user.id))
self.bulk_remove_objects_from_store(book_quotations, self.stream_id(user.id))
self.bulk_remove_objects_from_store(book_reviews, self.stream_id(user.id))
self.bulk_remove_objects_from_store(book_mentions, self.stream_id(user.id))
# determine which streams are enabled in settings.py # determine which streams are enabled in settings.py
@ -298,6 +329,11 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
remove_status_task.delay(instance.id) remove_status_task.delay(instance.id)
return return
# We don't want to create multiple add_status_tasks for each status, and because
# the transactions are atomic, on_commit won't run until the status is ready to add.
if not created:
return
# when creating new things, gotta wait on the transaction # when creating new things, gotta wait on the transaction
transaction.on_commit( transaction.on_commit(
lambda: add_status_on_create_command(sender, instance, created) lambda: add_status_on_create_command(sender, instance, created)
@ -306,13 +342,21 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
def add_status_on_create_command(sender, instance, created): def add_status_on_create_command(sender, instance, created):
"""runs this code only after the database commit completes""" """runs this code only after the database commit completes"""
priority = HIGH # boosts trigger 'saves" twice, so don't bother duplicating the task
if sender == models.Boost and not created:
return
priority = STREAMS
# check if this is an old status, de-prioritize if so # check if this is an old status, de-prioritize if so
# (this will happen if federation is very slow, or, more expectedly, on csv import) # (this will happen if federation is very slow, or, more expectedly, on csv import)
if instance.published_date < timezone.now() - timedelta( if instance.published_date < timezone.now() - timedelta(
days=1 days=1
) or instance.created_date < instance.published_date - timedelta(days=1): ) or instance.created_date < instance.published_date - timedelta(days=1):
priority = LOW # a backdated status from a local user is an import, don't add it
if instance.user.local:
return
# an out of date remote status is a low priority but should be added
priority = IMPORT_TRIGGERED
add_status_task.apply_async( add_status_task.apply_async(
args=(instance.id,), args=(instance.id,),
@ -456,7 +500,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs):
# ---- TASKS # ---- TASKS
@app.task(queue=LOW) @app.task(queue=STREAMS)
def add_book_statuses_task(user_id, book_id): def add_book_statuses_task(user_id, book_id):
"""add statuses related to a book on shelve""" """add statuses related to a book on shelve"""
user = models.User.objects.get(id=user_id) user = models.User.objects.get(id=user_id)
@ -464,7 +508,7 @@ def add_book_statuses_task(user_id, book_id):
BooksStream().add_book_statuses(user, book) BooksStream().add_book_statuses(user, book)
@app.task(queue=LOW) @app.task(queue=STREAMS)
def remove_book_statuses_task(user_id, book_id): def remove_book_statuses_task(user_id, book_id):
"""remove statuses about a book from a user's books feed""" """remove statuses about a book from a user's books feed"""
user = models.User.objects.get(id=user_id) user = models.User.objects.get(id=user_id)
@ -472,7 +516,7 @@ def remove_book_statuses_task(user_id, book_id):
BooksStream().remove_book_statuses(user, book) BooksStream().remove_book_statuses(user, book)
@app.task(queue=MEDIUM) @app.task(queue=STREAMS)
def populate_stream_task(stream, user_id): def populate_stream_task(stream, user_id):
"""background task for populating an empty activitystream""" """background task for populating an empty activitystream"""
user = models.User.objects.get(id=user_id) user = models.User.objects.get(id=user_id)
@ -480,7 +524,7 @@ def populate_stream_task(stream, user_id):
stream.populate_streams(user) stream.populate_streams(user)
@app.task(queue=MEDIUM) @app.task(queue=STREAMS)
def remove_status_task(status_ids): def remove_status_task(status_ids):
"""remove a status from any stream it might be in""" """remove a status from any stream it might be in"""
# this can take an id or a list of ids # this can take an id or a list of ids
@ -490,10 +534,12 @@ def remove_status_task(status_ids):
for stream in streams.values(): for stream in streams.values():
for status in statuses: for status in statuses:
stream.remove_object_from_related_stores(status) stream.remove_object_from_stores(
status, stream.get_stores_for_users(stream.get_audience(status))
)
@app.task(queue=HIGH) @app.task(queue=STREAMS)
def add_status_task(status_id, increment_unread=False): def add_status_task(status_id, increment_unread=False):
"""add a status to any stream it should be in""" """add a status to any stream it should be in"""
status = models.Status.objects.select_subclasses().get(id=status_id) status = models.Status.objects.select_subclasses().get(id=status_id)
@ -505,7 +551,7 @@ def add_status_task(status_id, increment_unread=False):
stream.add_status(status, increment_unread=increment_unread) stream.add_status(status, increment_unread=increment_unread)
@app.task(queue=MEDIUM) @app.task(queue=STREAMS)
def remove_user_statuses_task(viewer_id, user_id, stream_list=None): def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
"""remove all statuses by a user from a viewer's stream""" """remove all statuses by a user from a viewer's stream"""
stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@ -515,7 +561,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
stream.remove_user_statuses(viewer, user) stream.remove_user_statuses(viewer, user)
@app.task(queue=MEDIUM) @app.task(queue=STREAMS)
def add_user_statuses_task(viewer_id, user_id, stream_list=None): def add_user_statuses_task(viewer_id, user_id, stream_list=None):
"""add all statuses by a user to a viewer's stream""" """add all statuses by a user to a viewer's stream"""
stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@ -525,7 +571,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None):
stream.add_user_statuses(viewer, user) stream.add_user_statuses(viewer, user)
@app.task(queue=MEDIUM) @app.task(queue=STREAMS)
def handle_boost_task(boost_id): def handle_boost_task(boost_id):
"""remove the original post and other, earlier boosts""" """remove the original post and other, earlier boosts"""
instance = models.Status.objects.get(id=boost_id) instance = models.Status.objects.get(id=boost_id)
@ -539,10 +585,10 @@ def handle_boost_task(boost_id):
for stream in streams.values(): for stream in streams.values():
# people who should see the boost (not people who see the original status) # people who should see the boost (not people who see the original status)
audience = stream.get_stores_for_object(instance) audience = stream.get_stores_for_users(stream.get_audience(instance))
stream.remove_object_from_related_stores(boosted, stores=audience) stream.remove_object_from_stores(boosted, audience)
for status in old_versions: for status in old_versions:
stream.remove_object_from_related_stores(status, stores=audience) stream.remove_object_from_stores(status, audience)
def get_status_type(status): def get_status_type(status):

View file

@ -35,11 +35,12 @@ class BookwyrmConfig(AppConfig):
# pylint: disable=no-self-use # pylint: disable=no-self-use
def ready(self): def ready(self):
"""set up OTLP and preview image files, if desired""" """set up OTLP and preview image files, if desired"""
if settings.OTEL_EXPORTER_OTLP_ENDPOINT: if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE:
# pylint: disable=import-outside-toplevel # pylint: disable=import-outside-toplevel
from bookwyrm.telemetry import open_telemetry from bookwyrm.telemetry import open_telemetry
open_telemetry.instrumentDjango() open_telemetry.instrumentDjango()
open_telemetry.instrumentPostgres()
if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS: if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS:
# Download any fonts that we don't have yet # Download any fonts that we don't have yet

View file

@ -1,24 +1,62 @@
""" using a bookwyrm instance as a source of book data """ """ using a bookwyrm instance as a source of book data """
from __future__ import annotations
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from functools import reduce from functools import reduce
import operator import operator
from typing import Optional, Union, Any, Literal, overload
from django.contrib.postgres.search import SearchRank, SearchQuery from django.contrib.postgres.search import SearchRank, SearchQuery
from django.db.models import OuterRef, Subquery, F, Q from django.db.models import F, Q
from django.db.models.query import QuerySet
from bookwyrm import models from bookwyrm import models
from bookwyrm import connectors from bookwyrm import connectors
from bookwyrm.settings import MEDIA_FULL_URL from bookwyrm.settings import MEDIA_FULL_URL
@overload
def search(
query: str,
*,
min_confidence: float = 0,
filters: Optional[list[Any]] = None,
return_first: Literal[False],
) -> QuerySet[models.Edition]:
...
@overload
def search(
query: str,
*,
min_confidence: float = 0,
filters: Optional[list[Any]] = None,
return_first: Literal[True],
) -> Optional[models.Edition]:
...
# pylint: disable=arguments-differ # pylint: disable=arguments-differ
def search(query, min_confidence=0, filters=None, return_first=False): def search(
query: str,
*,
min_confidence: float = 0,
filters: Optional[list[Any]] = None,
return_first: bool = False,
) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
"""search your local database""" """search your local database"""
filters = filters or [] filters = filters or []
if not query: if not query:
return [] return None if return_first else []
# first, try searching unqiue identifiers query = query.strip()
results = search_identifiers(query, *filters, return_first=return_first)
results = None
# first, try searching unique identifiers
# unique identifiers never have spaces, title/author usually do
if not " " in query:
results = search_identifiers(query, *filters, return_first=return_first)
# if there were no identifier results...
if not results: if not results:
# then try searching title/author # then try searching title/author
results = search_title_author( results = search_title_author(
@ -35,24 +73,10 @@ def isbn_search(query):
# If the ISBN has only 9 characters, prepend missing zero # If the ISBN has only 9 characters, prepend missing zero
query = query.strip().upper().rjust(10, "0") query = query.strip().upper().rjust(10, "0")
filters = [{f: query} for f in ["isbn_10", "isbn_13"]] filters = [{f: query} for f in ["isbn_10", "isbn_13"]]
results = models.Edition.objects.filter( return models.Edition.objects.filter(
reduce(operator.or_, (Q(**f) for f in filters)) reduce(operator.or_, (Q(**f) for f in filters))
).distinct() ).distinct()
# when there are multiple editions of the same work, pick the default.
# it would be odd for this to happen.
default_editions = models.Edition.objects.filter(
parent_work=OuterRef("parent_work")
).order_by("-edition_rank")
results = (
results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
default_id=F("id")
)
or results
)
return results
def format_search_result(search_result): def format_search_result(search_result):
"""convert a book object into a search result object""" """convert a book object into a search result object"""
@ -73,7 +97,9 @@ def format_search_result(search_result):
).json() ).json()
def search_identifiers(query, *filters, return_first=False): def search_identifiers(
query, *filters, return_first=False
) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
"""tries remote_id, isbn; defined as dedupe fields on the model""" """tries remote_id, isbn; defined as dedupe fields on the model"""
if connectors.maybe_isbn(query): if connectors.maybe_isbn(query):
# Oh did you think the 'S' in ISBN stood for 'standard'? # Oh did you think the 'S' in ISBN stood for 'standard'?
@ -88,28 +114,15 @@ def search_identifiers(query, *filters, return_first=False):
results = models.Edition.objects.filter( results = models.Edition.objects.filter(
*filters, reduce(operator.or_, (Q(**f) for f in or_filters)) *filters, reduce(operator.or_, (Q(**f) for f in or_filters))
).distinct() ).distinct()
if results.count() <= 1:
if return_first:
return results.first()
return results
# when there are multiple editions of the same work, pick the default.
# it would be odd for this to happen.
default_editions = models.Edition.objects.filter(
parent_work=OuterRef("parent_work")
).order_by("-edition_rank")
results = (
results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
default_id=F("id")
)
or results
)
if return_first: if return_first:
return results.first() return results.first()
return results return results
def search_title_author(query, min_confidence, *filters, return_first=False): def search_title_author(
query, min_confidence, *filters, return_first=False
) -> QuerySet[models.Edition]:
"""searches for title and author""" """searches for title and author"""
query = SearchQuery(query, config="simple") | SearchQuery(query, config="english") query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
results = ( results = (
@ -120,19 +133,16 @@ def search_title_author(query, min_confidence, *filters, return_first=False):
) )
# when there are multiple editions of the same work, pick the closest # when there are multiple editions of the same work, pick the closest
editions_of_work = results.values("parent_work__id").values_list("parent_work__id") editions_of_work = results.values_list("parent_work__id", flat=True).distinct()
# filter out multiple editions of the same work # filter out multiple editions of the same work
list_results = [] list_results = []
for work_id in set(editions_of_work): for work_id in set(editions_of_work[:30]):
editions = results.filter(parent_work=work_id) result = (
default = editions.order_by("-edition_rank").first() results.filter(parent_work=work_id)
default_rank = default.rank if default else 0 .order_by("-rank", "-edition_rank")
# if mutliple books have the top rank, pick the default edition .first()
if default_rank == editions.first().rank: )
result = default
else:
result = editions.first()
if return_first: if return_first:
return result return result
@ -147,11 +157,11 @@ class SearchResult:
title: str title: str
key: str key: str
connector: object connector: object
view_link: str = None view_link: Optional[str] = None
author: str = None author: Optional[str] = None
year: str = None year: Optional[str] = None
cover: str = None cover: Optional[str] = None
confidence: int = 1 confidence: float = 1.0
def __repr__(self): def __repr__(self):
# pylint: disable=consider-using-f-string # pylint: disable=consider-using-f-string

View file

@ -1,44 +1,55 @@
""" functionality outline for a book data connector """ """ functionality outline for a book data connector """
from __future__ import annotations
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Optional, TypedDict, Any, Callable, Union, Iterator
from urllib.parse import quote_plus
import imghdr import imghdr
import logging import logging
import re import re
import asyncio
import requests
from requests.exceptions import RequestException
import aiohttp
from django.core.files.base import ContentFile from django.core.files.base import ContentFile
from django.db import transaction from django.db import transaction
import requests
from requests.exceptions import RequestException
from bookwyrm import activitypub, models, settings from bookwyrm import activitypub, models, settings
from bookwyrm.settings import USER_AGENT
from .connector_manager import load_more_data, ConnectorException, raise_not_valid_url from .connector_manager import load_more_data, ConnectorException, raise_not_valid_url
from .format_mappings import format_mappings from .format_mappings import format_mappings
from ..book_search import SearchResult
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
JsonDict = dict[str, Any]
class ConnectorResults(TypedDict):
"""TypedDict for results returned by connector"""
connector: AbstractMinimalConnector
results: list[SearchResult]
class AbstractMinimalConnector(ABC): class AbstractMinimalConnector(ABC):
"""just the bare bones, for other bookwyrm instances""" """just the bare bones, for other bookwyrm instances"""
def __init__(self, identifier): def __init__(self, identifier: str):
# load connector settings # load connector settings
info = models.Connector.objects.get(identifier=identifier) info = models.Connector.objects.get(identifier=identifier)
self.connector = info self.connector = info
# the things in the connector model to copy over # the things in the connector model to copy over
self_fields = [ self.base_url = info.base_url
"base_url", self.books_url = info.books_url
"books_url", self.covers_url = info.covers_url
"covers_url", self.search_url = info.search_url
"search_url", self.isbn_search_url = info.isbn_search_url
"isbn_search_url", self.name = info.name
"name", self.identifier = info.identifier
"identifier",
]
for field in self_fields:
setattr(self, field, getattr(info, field))
def get_search_url(self, query): def get_search_url(self, query: str) -> str:
"""format the query url""" """format the query url"""
# Check if the query resembles an ISBN # Check if the query resembles an ISBN
if maybe_isbn(query) and self.isbn_search_url and self.isbn_search_url != "": if maybe_isbn(query) and self.isbn_search_url and self.isbn_search_url != "":
@ -48,44 +59,93 @@ class AbstractMinimalConnector(ABC):
return f"{self.isbn_search_url}{normalized_query}" return f"{self.isbn_search_url}{normalized_query}"
# NOTE: previously, we tried searching isbn and if that produces no results, # NOTE: previously, we tried searching isbn and if that produces no results,
# searched as free text. This, instead, only searches isbn if it's isbn-y # searched as free text. This, instead, only searches isbn if it's isbn-y
return f"{self.search_url}{query}" return f"{self.search_url}{quote_plus(query)}"
def process_search_response(self, query, data, min_confidence): def process_search_response(
"""Format the search results based on the formt of the query""" self, query: str, data: Any, min_confidence: float
) -> list[SearchResult]:
"""Format the search results based on the format of the query"""
if maybe_isbn(query): if maybe_isbn(query):
return list(self.parse_isbn_search_data(data))[:10] return list(self.parse_isbn_search_data(data))[:10]
return list(self.parse_search_data(data, min_confidence))[:10] return list(self.parse_search_data(data, min_confidence))[:10]
async def get_results(
self,
session: aiohttp.ClientSession,
url: str,
min_confidence: float,
query: str,
) -> Optional[ConnectorResults]:
"""try this specific connector"""
# pylint: disable=line-too-long
headers = {
"Accept": (
'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
),
"User-Agent": USER_AGENT,
}
params = {"min_confidence": min_confidence}
try:
async with session.get(url, headers=headers, params=params) as response:
if not response.ok:
logger.info("Unable to connect to %s: %s", url, response.reason)
return None
try:
raw_data = await response.json()
except aiohttp.client_exceptions.ContentTypeError as err:
logger.exception(err)
return None
return ConnectorResults(
connector=self,
results=self.process_search_response(
query, raw_data, min_confidence
),
)
except asyncio.TimeoutError:
logger.info("Connection timed out for url: %s", url)
except aiohttp.ClientError as err:
logger.info(err)
return None
@abstractmethod @abstractmethod
def get_or_create_book(self, remote_id): def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
"""pull up a book record by whatever means possible""" """pull up a book record by whatever means possible"""
@abstractmethod @abstractmethod
def parse_search_data(self, data, min_confidence): def parse_search_data(
self, data: Any, min_confidence: float
) -> Iterator[SearchResult]:
"""turn the result json from a search into a list""" """turn the result json from a search into a list"""
@abstractmethod @abstractmethod
def parse_isbn_search_data(self, data): def parse_isbn_search_data(self, data: Any) -> Iterator[SearchResult]:
"""turn the result json from a search into a list""" """turn the result json from a search into a list"""
class AbstractConnector(AbstractMinimalConnector): class AbstractConnector(AbstractMinimalConnector):
"""generic book data connector""" """generic book data connector"""
def __init__(self, identifier): generated_remote_link_field = ""
def __init__(self, identifier: str):
super().__init__(identifier) super().__init__(identifier)
# fields we want to look for in book data to copy over # fields we want to look for in book data to copy over
# title we handle separately. # title we handle separately.
self.book_mappings = [] self.book_mappings: list[Mapping] = []
self.author_mappings: list[Mapping] = []
def get_or_create_book(self, remote_id): def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
"""translate arbitrary json into an Activitypub dataclass""" """translate arbitrary json into an Activitypub dataclass"""
# first, check if we have the origin_id saved # first, check if we have the origin_id saved
existing = models.Edition.find_existing_by_remote_id( existing = models.Edition.find_existing_by_remote_id(
remote_id remote_id
) or models.Work.find_existing_by_remote_id(remote_id) ) or models.Work.find_existing_by_remote_id(remote_id)
if existing: if existing:
if hasattr(existing, "default_edition"): if hasattr(existing, "default_edition") and isinstance(
existing.default_edition, models.Edition
):
return existing.default_edition return existing.default_edition
return existing return existing
@ -117,6 +177,9 @@ class AbstractConnector(AbstractMinimalConnector):
) )
# this will dedupe automatically # this will dedupe automatically
work = work_activity.to_model(model=models.Work, overwrite=False) work = work_activity.to_model(model=models.Work, overwrite=False)
if not work:
return None
for author in self.get_authors_from_data(work_data): for author in self.get_authors_from_data(work_data):
work.authors.add(author) work.authors.add(author)
@ -124,12 +187,21 @@ class AbstractConnector(AbstractMinimalConnector):
load_more_data.delay(self.connector.id, work.id) load_more_data.delay(self.connector.id, work.id)
return edition return edition
def get_book_data(self, remote_id): # pylint: disable=no-self-use def get_book_data(self, remote_id: str) -> JsonDict: # pylint: disable=no-self-use
"""this allows connectors to override the default behavior""" """this allows connectors to override the default behavior"""
return get_data(remote_id) return get_data(remote_id)
def create_edition_from_data(self, work, edition_data, instance=None): def create_edition_from_data(
self,
work: models.Work,
edition_data: Union[str, JsonDict],
instance: Optional[models.Edition] = None,
) -> Optional[models.Edition]:
"""if we already have the work, we're ready""" """if we already have the work, we're ready"""
if isinstance(edition_data, str):
# We don't expect a string here
return None
mapped_data = dict_from_mappings(edition_data, self.book_mappings) mapped_data = dict_from_mappings(edition_data, self.book_mappings)
mapped_data["work"] = work.remote_id mapped_data["work"] = work.remote_id
edition_activity = activitypub.Edition(**mapped_data) edition_activity = activitypub.Edition(**mapped_data)
@ -137,6 +209,9 @@ class AbstractConnector(AbstractMinimalConnector):
model=models.Edition, overwrite=False, instance=instance model=models.Edition, overwrite=False, instance=instance
) )
if not edition:
return None
# if we're updating an existing instance, we don't need to load authors # if we're updating an existing instance, we don't need to load authors
if instance: if instance:
return edition return edition
@ -153,7 +228,9 @@ class AbstractConnector(AbstractMinimalConnector):
return edition return edition
def get_or_create_author(self, remote_id, instance=None): def get_or_create_author(
self, remote_id: str, instance: Optional[models.Author] = None
) -> Optional[models.Author]:
"""load that author""" """load that author"""
if not instance: if not instance:
existing = models.Author.find_existing_by_remote_id(remote_id) existing = models.Author.find_existing_by_remote_id(remote_id)
@ -173,46 +250,51 @@ class AbstractConnector(AbstractMinimalConnector):
model=models.Author, overwrite=False, instance=instance model=models.Author, overwrite=False, instance=instance
) )
def get_remote_id_from_model(self, obj): def get_remote_id_from_model(self, obj: models.BookDataModel) -> Optional[str]:
"""given the data stored, how can we look this up""" """given the data stored, how can we look this up"""
return getattr(obj, getattr(self, "generated_remote_link_field")) remote_id: Optional[str] = getattr(obj, self.generated_remote_link_field)
return remote_id
def update_author_from_remote(self, obj): def update_author_from_remote(self, obj: models.Author) -> Optional[models.Author]:
"""load the remote data from this connector and add it to an existing author""" """load the remote data from this connector and add it to an existing author"""
remote_id = self.get_remote_id_from_model(obj) remote_id = self.get_remote_id_from_model(obj)
if not remote_id:
return None
return self.get_or_create_author(remote_id, instance=obj) return self.get_or_create_author(remote_id, instance=obj)
def update_book_from_remote(self, obj): def update_book_from_remote(self, obj: models.Edition) -> Optional[models.Edition]:
"""load the remote data from this connector and add it to an existing book""" """load the remote data from this connector and add it to an existing book"""
remote_id = self.get_remote_id_from_model(obj) remote_id = self.get_remote_id_from_model(obj)
if not remote_id:
return None
data = self.get_book_data(remote_id) data = self.get_book_data(remote_id)
return self.create_edition_from_data(obj.parent_work, data, instance=obj) return self.create_edition_from_data(obj.parent_work, data, instance=obj)
@abstractmethod @abstractmethod
def is_work_data(self, data): def is_work_data(self, data: JsonDict) -> bool:
"""differentiate works and editions""" """differentiate works and editions"""
@abstractmethod @abstractmethod
def get_edition_from_work_data(self, data): def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
"""every work needs at least one edition""" """every work needs at least one edition"""
@abstractmethod @abstractmethod
def get_work_from_edition_data(self, data): def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
"""every edition needs a work""" """every edition needs a work"""
@abstractmethod @abstractmethod
def get_authors_from_data(self, data): def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
"""load author data""" """load author data"""
@abstractmethod @abstractmethod
def expand_book_data(self, book): def expand_book_data(self, book: models.Book) -> None:
"""get more info on a book""" """get more info on a book"""
def dict_from_mappings(data, mappings): def dict_from_mappings(data: JsonDict, mappings: list[Mapping]) -> JsonDict:
"""create a dict in Activitypub format, using mappings supplies by """create a dict in Activitypub format, using mappings supplies by
the subclass""" the subclass"""
result = {} result: JsonDict = {}
for mapping in mappings: for mapping in mappings:
# sometimes there are multiple mappings for one field, don't # sometimes there are multiple mappings for one field, don't
# overwrite earlier writes in that case # overwrite earlier writes in that case
@ -222,7 +304,11 @@ def dict_from_mappings(data, mappings):
return result return result
def get_data(url, params=None, timeout=10): def get_data(
url: str,
params: Optional[dict[str, str]] = None,
timeout: int = settings.QUERY_TIMEOUT,
) -> JsonDict:
"""wrapper for request.get""" """wrapper for request.get"""
# check if the url is blocked # check if the url is blocked
raise_not_valid_url(url) raise_not_valid_url(url)
@ -244,17 +330,26 @@ def get_data(url, params=None, timeout=10):
raise ConnectorException(err) raise ConnectorException(err)
if not resp.ok: if not resp.ok:
raise ConnectorException() if resp.status_code == 401:
# this is probably an AUTHORIZED_FETCH issue
resp.raise_for_status()
else:
raise ConnectorException()
try: try:
data = resp.json() data = resp.json()
except ValueError as err: except ValueError as err:
logger.info(err) logger.info(err)
raise ConnectorException(err) raise ConnectorException(err)
if not isinstance(data, dict):
raise ConnectorException("Unexpected data format")
return data return data
def get_image(url, timeout=10): def get_image(
url: str, timeout: int = 10
) -> Union[tuple[ContentFile[bytes], str], tuple[None, None]]:
"""wrapper for requesting an image""" """wrapper for requesting an image"""
raise_not_valid_url(url) raise_not_valid_url(url)
try: try:
@ -284,14 +379,19 @@ def get_image(url, timeout=10):
class Mapping: class Mapping:
"""associate a local database field with a field in an external dataset""" """associate a local database field with a field in an external dataset"""
def __init__(self, local_field, remote_field=None, formatter=None): def __init__(
self,
local_field: str,
remote_field: Optional[str] = None,
formatter: Optional[Callable[[Any], Any]] = None,
):
noop = lambda x: x noop = lambda x: x
self.local_field = local_field self.local_field = local_field
self.remote_field = remote_field or local_field self.remote_field = remote_field or local_field
self.formatter = formatter or noop self.formatter = formatter or noop
def get_value(self, data): def get_value(self, data: JsonDict) -> Optional[Any]:
"""pull a field from incoming json and return the formatted version""" """pull a field from incoming json and return the formatted version"""
value = data.get(self.remote_field) value = data.get(self.remote_field)
if not value: if not value:
@ -302,7 +402,7 @@ class Mapping:
return None return None
def infer_physical_format(format_text): def infer_physical_format(format_text: str) -> Optional[str]:
"""try to figure out what the standardized format is from the free value""" """try to figure out what the standardized format is from the free value"""
format_text = format_text.lower() format_text = format_text.lower()
if format_text in format_mappings: if format_text in format_mappings:
@ -315,8 +415,8 @@ def infer_physical_format(format_text):
return matches[0] return matches[0]
def unique_physical_format(format_text): def unique_physical_format(format_text: str) -> Optional[str]:
"""only store the format if it isn't diretly in the format mappings""" """only store the format if it isn't directly in the format mappings"""
format_text = format_text.lower() format_text = format_text.lower()
if format_text in format_mappings: if format_text in format_mappings:
# try a direct match, so saving this would be redundant # try a direct match, so saving this would be redundant
@ -324,7 +424,7 @@ def unique_physical_format(format_text):
return format_text return format_text
def maybe_isbn(query): def maybe_isbn(query: str) -> bool:
"""check if a query looks like an isbn""" """check if a query looks like an isbn"""
isbn = re.sub(r"[\W_]", "", query) # removes filler characters isbn = re.sub(r"[\W_]", "", query) # removes filler characters
# ISBNs must be numeric except an ISBN10 checkdigit can be 'X' # ISBNs must be numeric except an ISBN10 checkdigit can be 'X'

View file

@ -1,4 +1,7 @@
""" using another bookwyrm instance as a source of book data """ """ using another bookwyrm instance as a source of book data """
from __future__ import annotations
from typing import Any, Iterator
from bookwyrm import activitypub, models from bookwyrm import activitypub, models
from bookwyrm.book_search import SearchResult from bookwyrm.book_search import SearchResult
from .abstract_connector import AbstractMinimalConnector from .abstract_connector import AbstractMinimalConnector
@ -7,15 +10,19 @@ from .abstract_connector import AbstractMinimalConnector
class Connector(AbstractMinimalConnector): class Connector(AbstractMinimalConnector):
"""this is basically just for search""" """this is basically just for search"""
def get_or_create_book(self, remote_id): def get_or_create_book(self, remote_id: str) -> models.Edition:
return activitypub.resolve_remote_id(remote_id, model=models.Edition) return activitypub.resolve_remote_id(remote_id, model=models.Edition)
def parse_search_data(self, data, min_confidence): def parse_search_data(
self, data: list[dict[str, Any]], min_confidence: float
) -> Iterator[SearchResult]:
for search_result in data: for search_result in data:
search_result["connector"] = self search_result["connector"] = self
yield SearchResult(**search_result) yield SearchResult(**search_result)
def parse_isbn_search_data(self, data): def parse_isbn_search_data(
self, data: list[dict[str, Any]]
) -> Iterator[SearchResult]:
for search_result in data: for search_result in data:
search_result["connector"] = self search_result["connector"] = self
yield SearchResult(**search_result) yield SearchResult(**search_result)

View file

@ -1,8 +1,11 @@
""" interface with whatever connectors the app has """ """ interface with whatever connectors the app has """
from __future__ import annotations
import asyncio import asyncio
import importlib import importlib
import ipaddress import ipaddress
import logging import logging
from asyncio import Future
from typing import Iterator, Any, Optional, Union, overload, Literal
from urllib.parse import urlparse from urllib.parse import urlparse
import aiohttp import aiohttp
@ -12,8 +15,10 @@ from django.db.models import signals
from requests import HTTPError from requests import HTTPError
from bookwyrm import book_search, models from bookwyrm import book_search, models
from bookwyrm.settings import SEARCH_TIMEOUT, USER_AGENT from bookwyrm.book_search import SearchResult
from bookwyrm.tasks import app, LOW from bookwyrm.connectors import abstract_connector
from bookwyrm.settings import SEARCH_TIMEOUT
from bookwyrm.tasks import app, CONNECTORS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -22,61 +27,46 @@ class ConnectorException(HTTPError):
"""when the connector can't do what was asked""" """when the connector can't do what was asked"""
async def get_results(session, url, min_confidence, query, connector): async def async_connector_search(
"""try this specific connector""" query: str,
# pylint: disable=line-too-long items: list[tuple[str, abstract_connector.AbstractConnector]],
headers = { min_confidence: float,
"Accept": ( ) -> list[Optional[abstract_connector.ConnectorResults]]:
'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
),
"User-Agent": USER_AGENT,
}
params = {"min_confidence": min_confidence}
try:
async with session.get(url, headers=headers, params=params) as response:
if not response.ok:
logger.info("Unable to connect to %s: %s", url, response.reason)
return
try:
raw_data = await response.json()
except aiohttp.client_exceptions.ContentTypeError as err:
logger.exception(err)
return
return {
"connector": connector,
"results": connector.process_search_response(
query, raw_data, min_confidence
),
}
except asyncio.TimeoutError:
logger.info("Connection timed out for url: %s", url)
except aiohttp.ClientError as err:
logger.info(err)
async def async_connector_search(query, items, min_confidence):
"""Try a number of requests simultaneously""" """Try a number of requests simultaneously"""
timeout = aiohttp.ClientTimeout(total=SEARCH_TIMEOUT) timeout = aiohttp.ClientTimeout(total=SEARCH_TIMEOUT)
async with aiohttp.ClientSession(timeout=timeout) as session: async with aiohttp.ClientSession(timeout=timeout) as session:
tasks = [] tasks: list[Future[Optional[abstract_connector.ConnectorResults]]] = []
for url, connector in items: for url, connector in items:
tasks.append( tasks.append(
asyncio.ensure_future( asyncio.ensure_future(
get_results(session, url, min_confidence, query, connector) connector.get_results(session, url, min_confidence, query)
) )
) )
results = await asyncio.gather(*tasks) results = await asyncio.gather(*tasks)
return results return list(results)
def search(query, min_confidence=0.1, return_first=False): @overload
"""find books based on arbitary keywords""" def search(
query: str, *, min_confidence: float = 0.1, return_first: Literal[False]
) -> list[abstract_connector.ConnectorResults]:
...
@overload
def search(
query: str, *, min_confidence: float = 0.1, return_first: Literal[True]
) -> Optional[SearchResult]:
...
def search(
query: str, *, min_confidence: float = 0.1, return_first: bool = False
) -> Union[list[abstract_connector.ConnectorResults], Optional[SearchResult]]:
"""find books based on arbitrary keywords"""
if not query: if not query:
return [] return None if return_first else []
results = []
items = [] items = []
for connector in get_connectors(): for connector in get_connectors():
@ -91,8 +81,12 @@ def search(query, min_confidence=0.1, return_first=False):
items.append((url, connector)) items.append((url, connector))
# load as many results as we can # load as many results as we can
results = asyncio.run(async_connector_search(query, items, min_confidence)) # failed requests will return None, so filter those out
results = [r for r in results if r] results = [
r
for r in asyncio.run(async_connector_search(query, items, min_confidence))
if r
]
if return_first: if return_first:
# find the best result from all the responses and return that # find the best result from all the responses and return that
@ -100,11 +94,12 @@ def search(query, min_confidence=0.1, return_first=False):
all_results = sorted(all_results, key=lambda r: r.confidence, reverse=True) all_results = sorted(all_results, key=lambda r: r.confidence, reverse=True)
return all_results[0] if all_results else None return all_results[0] if all_results else None
# failed requests will return None, so filter those out
return results return results
def first_search_result(query, min_confidence=0.1): def first_search_result(
query: str, min_confidence: float = 0.1
) -> Union[models.Edition, SearchResult, None]:
"""search until you find a result that fits""" """search until you find a result that fits"""
# try local search first # try local search first
result = book_search.search(query, min_confidence=min_confidence, return_first=True) result = book_search.search(query, min_confidence=min_confidence, return_first=True)
@ -114,13 +109,13 @@ def first_search_result(query, min_confidence=0.1):
return search(query, min_confidence=min_confidence, return_first=True) or None return search(query, min_confidence=min_confidence, return_first=True) or None
def get_connectors(): def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
"""load all connectors""" """load all connectors"""
for info in models.Connector.objects.filter(active=True).order_by("priority").all(): for info in models.Connector.objects.filter(active=True).order_by("priority").all():
yield load_connector(info) yield load_connector(info)
def get_or_create_connector(remote_id): def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
"""get the connector related to the object's server""" """get the connector related to the object's server"""
url = urlparse(remote_id) url = urlparse(remote_id)
identifier = url.netloc identifier = url.netloc
@ -143,8 +138,8 @@ def get_or_create_connector(remote_id):
return load_connector(connector_info) return load_connector(connector_info)
@app.task(queue=LOW) @app.task(queue=CONNECTORS)
def load_more_data(connector_id, book_id): def load_more_data(connector_id: str, book_id: str) -> None:
"""background the work of getting all 10,000 editions of LoTR""" """background the work of getting all 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id) connector_info = models.Connector.objects.get(id=connector_id)
connector = load_connector(connector_info) connector = load_connector(connector_info)
@ -152,8 +147,10 @@ def load_more_data(connector_id, book_id):
connector.expand_book_data(book) connector.expand_book_data(book)
@app.task(queue=LOW) @app.task(queue=CONNECTORS)
def create_edition_task(connector_id, work_id, data): def create_edition_task(
connector_id: int, work_id: int, data: Union[str, abstract_connector.JsonDict]
) -> None:
"""separate task for each of the 10,000 editions of LoTR""" """separate task for each of the 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id) connector_info = models.Connector.objects.get(id=connector_id)
connector = load_connector(connector_info) connector = load_connector(connector_info)
@ -161,23 +158,31 @@ def create_edition_task(connector_id, work_id, data):
connector.create_edition_from_data(work, data) connector.create_edition_from_data(work, data)
def load_connector(connector_info): def load_connector(
connector_info: models.Connector,
) -> abstract_connector.AbstractConnector:
"""instantiate the connector class""" """instantiate the connector class"""
connector = importlib.import_module( connector = importlib.import_module(
f"bookwyrm.connectors.{connector_info.connector_file}" f"bookwyrm.connectors.{connector_info.connector_file}"
) )
return connector.Connector(connector_info.identifier) return connector.Connector(connector_info.identifier) # type: ignore[no-any-return]
@receiver(signals.post_save, sender="bookwyrm.FederatedServer") @receiver(signals.post_save, sender="bookwyrm.FederatedServer")
# pylint: disable=unused-argument # pylint: disable=unused-argument
def create_connector(sender, instance, created, *args, **kwargs): def create_connector(
sender: Any,
instance: models.FederatedServer,
created: Any,
*args: Any,
**kwargs: Any,
) -> None:
"""create a connector to an external bookwyrm server""" """create a connector to an external bookwyrm server"""
if instance.application_type == "bookwyrm": if instance.application_type == "bookwyrm":
get_or_create_connector(f"https://{instance.server_name}") get_or_create_connector(f"https://{instance.server_name}")
def raise_not_valid_url(url): def raise_not_valid_url(url: str) -> None:
"""do some basic reality checks on the url""" """do some basic reality checks on the url"""
parsed = urlparse(url) parsed = urlparse(url)
if not parsed.scheme in ["http", "https"]: if not parsed.scheme in ["http", "https"]:

View file

@ -1,9 +1,10 @@
""" inventaire data connector """ """ inventaire data connector """
import re import re
from typing import Any, Union, Optional, Iterator, Iterable
from bookwyrm import models from bookwyrm import models
from bookwyrm.book_search import SearchResult from bookwyrm.book_search import SearchResult
from .abstract_connector import AbstractConnector, Mapping from .abstract_connector import AbstractConnector, Mapping, JsonDict
from .abstract_connector import get_data from .abstract_connector import get_data
from .connector_manager import ConnectorException, create_edition_task from .connector_manager import ConnectorException, create_edition_task
@ -13,7 +14,7 @@ class Connector(AbstractConnector):
generated_remote_link_field = "inventaire_id" generated_remote_link_field = "inventaire_id"
def __init__(self, identifier): def __init__(self, identifier: str):
super().__init__(identifier) super().__init__(identifier)
get_first = lambda a: a[0] get_first = lambda a: a[0]
@ -60,13 +61,13 @@ class Connector(AbstractConnector):
Mapping("died", remote_field="wdt:P570", formatter=get_first), Mapping("died", remote_field="wdt:P570", formatter=get_first),
] + shared_mappings ] + shared_mappings
def get_remote_id(self, value): def get_remote_id(self, value: str) -> str:
"""convert an id/uri into a url""" """convert an id/uri into a url"""
return f"{self.books_url}?action=by-uris&uris={value}" return f"{self.books_url}?action=by-uris&uris={value}"
def get_book_data(self, remote_id): def get_book_data(self, remote_id: str) -> JsonDict:
data = get_data(remote_id) data = get_data(remote_id)
extracted = list(data.get("entities").values()) extracted = list(data.get("entities", {}).values())
try: try:
data = extracted[0] data = extracted[0]
except (KeyError, IndexError): except (KeyError, IndexError):
@ -74,10 +75,16 @@ class Connector(AbstractConnector):
# flatten the data so that images, uri, and claims are on the same level # flatten the data so that images, uri, and claims are on the same level
return { return {
**data.get("claims", {}), **data.get("claims", {}),
**{k: data.get(k) for k in ["uri", "image", "labels", "sitelinks", "type"]}, **{
k: data.get(k)
for k in ["uri", "image", "labels", "sitelinks", "type"]
if k in data
},
} }
def parse_search_data(self, data, min_confidence): def parse_search_data(
self, data: JsonDict, min_confidence: float
) -> Iterator[SearchResult]:
for search_result in data.get("results", []): for search_result in data.get("results", []):
images = search_result.get("image") images = search_result.get("image")
cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None
@ -96,8 +103,8 @@ class Connector(AbstractConnector):
connector=self, connector=self,
) )
def parse_isbn_search_data(self, data): def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
"""got some daaaata""" """got some data"""
results = data.get("entities") results = data.get("entities")
if not results: if not results:
return return
@ -114,35 +121,44 @@ class Connector(AbstractConnector):
connector=self, connector=self,
) )
def is_work_data(self, data): def is_work_data(self, data: JsonDict) -> bool:
return data.get("type") == "work" return data.get("type") == "work"
def load_edition_data(self, work_uri): def load_edition_data(self, work_uri: str) -> JsonDict:
"""get a list of editions for a work""" """get a list of editions for a work"""
# pylint: disable=line-too-long # pylint: disable=line-too-long
url = f"{self.books_url}?action=reverse-claims&property=wdt:P629&value={work_uri}&sort=true" url = f"{self.books_url}?action=reverse-claims&property=wdt:P629&value={work_uri}&sort=true"
return get_data(url) return get_data(url)
def get_edition_from_work_data(self, data): def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
data = self.load_edition_data(data.get("uri")) work_uri = data.get("uri")
if not work_uri:
raise ConnectorException("Invalid URI")
data = self.load_edition_data(work_uri)
try: try:
uri = data.get("uris", [])[0] uri = data.get("uris", [])[0]
except IndexError: except IndexError:
raise ConnectorException("Invalid book data") raise ConnectorException("Invalid book data")
return self.get_book_data(self.get_remote_id(uri)) return self.get_book_data(self.get_remote_id(uri))
def get_work_from_edition_data(self, data): def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
uri = data.get("wdt:P629", [None])[0] try:
uri = data.get("wdt:P629", [])[0]
except IndexError:
raise ConnectorException("Invalid book data")
if not uri: if not uri:
raise ConnectorException("Invalid book data") raise ConnectorException("Invalid book data")
return self.get_book_data(self.get_remote_id(uri)) return self.get_book_data(self.get_remote_id(uri))
def get_authors_from_data(self, data): def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
authors = data.get("wdt:P50", []) authors = data.get("wdt:P50", [])
for author in authors: for author in authors:
yield self.get_or_create_author(self.get_remote_id(author)) model = self.get_or_create_author(self.get_remote_id(author))
if model:
yield model
def expand_book_data(self, book): def expand_book_data(self, book: models.Book) -> None:
work = book work = book
# go from the edition to the work, if necessary # go from the edition to the work, if necessary
if isinstance(book, models.Edition): if isinstance(book, models.Edition):
@ -154,36 +170,45 @@ class Connector(AbstractConnector):
# who knows, man # who knows, man
return return
for edition_uri in edition_options.get("uris"): for edition_uri in edition_options.get("uris", []):
remote_id = self.get_remote_id(edition_uri) remote_id = self.get_remote_id(edition_uri)
create_edition_task.delay(self.connector.id, work.id, remote_id) create_edition_task.delay(self.connector.id, work.id, remote_id)
def create_edition_from_data(self, work, edition_data, instance=None): def create_edition_from_data(
self,
work: models.Work,
edition_data: Union[str, JsonDict],
instance: Optional[models.Edition] = None,
) -> Optional[models.Edition]:
"""pass in the url as data and then call the version in abstract connector""" """pass in the url as data and then call the version in abstract connector"""
if isinstance(edition_data, str): if isinstance(edition_data, str):
try: try:
edition_data = self.get_book_data(edition_data) edition_data = self.get_book_data(edition_data)
except ConnectorException: except ConnectorException:
# who, indeed, knows # who, indeed, knows
return return None
super().create_edition_from_data(work, edition_data, instance=instance) return super().create_edition_from_data(work, edition_data, instance=instance)
def get_cover_url(self, cover_blob, *_): def get_cover_url(
self, cover_blob: Union[list[JsonDict], JsonDict], *_: Any
) -> Optional[str]:
"""format the relative cover url into an absolute one: """format the relative cover url into an absolute one:
{"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"} {"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"}
""" """
# covers may or may not be a list # covers may or may not be a list
if isinstance(cover_blob, list) and len(cover_blob) > 0: if isinstance(cover_blob, list):
if len(cover_blob) == 0:
return None
cover_blob = cover_blob[0] cover_blob = cover_blob[0]
cover_id = cover_blob.get("url") cover_id = cover_blob.get("url")
if not cover_id: if not isinstance(cover_id, str):
return None return None
# cover may or may not be an absolute url already # cover may or may not be an absolute url already
if re.match(r"^http", cover_id): if re.match(r"^http", cover_id):
return cover_id return cover_id
return f"{self.covers_url}{cover_id}" return f"{self.covers_url}{cover_id}"
def resolve_keys(self, keys): def resolve_keys(self, keys: Iterable[str]) -> list[str]:
"""cool, it's "wd:Q3156592" now what the heck does that mean""" """cool, it's "wd:Q3156592" now what the heck does that mean"""
results = [] results = []
for uri in keys: for uri in keys:
@ -191,10 +216,10 @@ class Connector(AbstractConnector):
data = self.get_book_data(self.get_remote_id(uri)) data = self.get_book_data(self.get_remote_id(uri))
except ConnectorException: except ConnectorException:
continue continue
results.append(get_language_code(data.get("labels"))) results.append(get_language_code(data.get("labels", {})))
return results return results
def get_description(self, links): def get_description(self, links: JsonDict) -> str:
"""grab an extracted excerpt from wikipedia""" """grab an extracted excerpt from wikipedia"""
link = links.get("enwiki") link = links.get("enwiki")
if not link: if not link:
@ -204,15 +229,15 @@ class Connector(AbstractConnector):
data = get_data(url) data = get_data(url)
except ConnectorException: except ConnectorException:
return "" return ""
return data.get("extract") return data.get("extract", "")
def get_remote_id_from_model(self, obj): def get_remote_id_from_model(self, obj: models.BookDataModel) -> str:
"""use get_remote_id to figure out the link from a model obj""" """use get_remote_id to figure out the link from a model obj"""
remote_id_value = obj.inventaire_id remote_id_value = obj.inventaire_id
return self.get_remote_id(remote_id_value) return self.get_remote_id(remote_id_value)
def get_language_code(options, code="en"): def get_language_code(options: JsonDict, code: str = "en") -> Any:
"""when there are a bunch of translation but we need a single field""" """when there are a bunch of translation but we need a single field"""
result = options.get(code) result = options.get(code)
if result: if result:

View file

@ -1,9 +1,13 @@
""" openlibrary data connector """ """ openlibrary data connector """
import re import re
from typing import Any, Optional, Union, Iterator, Iterable
from markdown import markdown
from bookwyrm import models from bookwyrm import models
from bookwyrm.book_search import SearchResult from bookwyrm.book_search import SearchResult
from .abstract_connector import AbstractConnector, Mapping from bookwyrm.utils.sanitizer import clean
from .abstract_connector import AbstractConnector, Mapping, JsonDict
from .abstract_connector import get_data, infer_physical_format, unique_physical_format from .abstract_connector import get_data, infer_physical_format, unique_physical_format
from .connector_manager import ConnectorException, create_edition_task from .connector_manager import ConnectorException, create_edition_task
from .openlibrary_languages import languages from .openlibrary_languages import languages
@ -14,7 +18,7 @@ class Connector(AbstractConnector):
generated_remote_link_field = "openlibrary_link" generated_remote_link_field = "openlibrary_link"
def __init__(self, identifier): def __init__(self, identifier: str):
super().__init__(identifier) super().__init__(identifier)
get_first = lambda a, *args: a[0] get_first = lambda a, *args: a[0]
@ -94,14 +98,14 @@ class Connector(AbstractConnector):
Mapping("inventaire_id", remote_field="links", formatter=get_inventaire_id), Mapping("inventaire_id", remote_field="links", formatter=get_inventaire_id),
] ]
def get_book_data(self, remote_id): def get_book_data(self, remote_id: str) -> JsonDict:
data = get_data(remote_id) data = get_data(remote_id)
if data.get("type", {}).get("key") == "/type/redirect": if data.get("type", {}).get("key") == "/type/redirect":
remote_id = self.base_url + data.get("location") remote_id = self.base_url + data.get("location", "")
return get_data(remote_id) return get_data(remote_id)
return data return data
def get_remote_id_from_data(self, data): def get_remote_id_from_data(self, data: JsonDict) -> str:
"""format a url from an openlibrary id field""" """format a url from an openlibrary id field"""
try: try:
key = data["key"] key = data["key"]
@ -109,10 +113,10 @@ class Connector(AbstractConnector):
raise ConnectorException("Invalid book data") raise ConnectorException("Invalid book data")
return f"{self.books_url}{key}" return f"{self.books_url}{key}"
def is_work_data(self, data): def is_work_data(self, data: JsonDict) -> bool:
return bool(re.match(r"^[\/\w]+OL\d+W$", data["key"])) return bool(re.match(r"^[\/\w]+OL\d+W$", data["key"]))
def get_edition_from_work_data(self, data): def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
try: try:
key = data["key"] key = data["key"]
except KeyError: except KeyError:
@ -124,7 +128,7 @@ class Connector(AbstractConnector):
raise ConnectorException("No editions for work") raise ConnectorException("No editions for work")
return edition return edition
def get_work_from_edition_data(self, data): def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
try: try:
key = data["works"][0]["key"] key = data["works"][0]["key"]
except (IndexError, KeyError): except (IndexError, KeyError):
@ -132,7 +136,7 @@ class Connector(AbstractConnector):
url = f"{self.books_url}{key}" url = f"{self.books_url}{key}"
return self.get_book_data(url) return self.get_book_data(url)
def get_authors_from_data(self, data): def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
"""parse author json and load or create authors""" """parse author json and load or create authors"""
for author_blob in data.get("authors", []): for author_blob in data.get("authors", []):
author_blob = author_blob.get("author", author_blob) author_blob = author_blob.get("author", author_blob)
@ -144,7 +148,7 @@ class Connector(AbstractConnector):
continue continue
yield author yield author
def get_cover_url(self, cover_blob, size="L"): def get_cover_url(self, cover_blob: list[str], size: str = "L") -> Optional[str]:
"""ask openlibrary for the cover""" """ask openlibrary for the cover"""
if not cover_blob: if not cover_blob:
return None return None
@ -152,8 +156,10 @@ class Connector(AbstractConnector):
image_name = f"{cover_id}-{size}.jpg" image_name = f"{cover_id}-{size}.jpg"
return f"{self.covers_url}/b/id/{image_name}" return f"{self.covers_url}/b/id/{image_name}"
def parse_search_data(self, data, min_confidence): def parse_search_data(
for idx, search_result in enumerate(data.get("docs")): self, data: JsonDict, min_confidence: float
) -> Iterator[SearchResult]:
for idx, search_result in enumerate(data.get("docs", [])):
# build the remote id from the openlibrary key # build the remote id from the openlibrary key
key = self.books_url + search_result["key"] key = self.books_url + search_result["key"]
author = search_result.get("author_name") or ["Unknown"] author = search_result.get("author_name") or ["Unknown"]
@ -174,7 +180,7 @@ class Connector(AbstractConnector):
confidence=confidence, confidence=confidence,
) )
def parse_isbn_search_data(self, data): def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
for search_result in list(data.values()): for search_result in list(data.values()):
# build the remote id from the openlibrary key # build the remote id from the openlibrary key
key = self.books_url + search_result["key"] key = self.books_url + search_result["key"]
@ -188,12 +194,12 @@ class Connector(AbstractConnector):
year=search_result.get("publish_date"), year=search_result.get("publish_date"),
) )
def load_edition_data(self, olkey): def load_edition_data(self, olkey: str) -> JsonDict:
"""query openlibrary for editions of a work""" """query openlibrary for editions of a work"""
url = f"{self.books_url}/works/{olkey}/editions" url = f"{self.books_url}/works/{olkey}/editions"
return self.get_book_data(url) return self.get_book_data(url)
def expand_book_data(self, book): def expand_book_data(self, book: models.Book) -> None:
work = book work = book
# go from the edition to the work, if necessary # go from the edition to the work, if necessary
if isinstance(book, models.Edition): if isinstance(book, models.Edition):
@ -206,14 +212,14 @@ class Connector(AbstractConnector):
# who knows, man # who knows, man
return return
for edition_data in edition_options.get("entries"): for edition_data in edition_options.get("entries", []):
# does this edition have ANY interesting data? # does this edition have ANY interesting data?
if ignore_edition(edition_data): if ignore_edition(edition_data):
continue continue
create_edition_task.delay(self.connector.id, work.id, edition_data) create_edition_task.delay(self.connector.id, work.id, edition_data)
def ignore_edition(edition_data): def ignore_edition(edition_data: JsonDict) -> bool:
"""don't load a million editions that have no metadata""" """don't load a million editions that have no metadata"""
# an isbn, we love to see it # an isbn, we love to see it
if edition_data.get("isbn_13") or edition_data.get("isbn_10"): if edition_data.get("isbn_13") or edition_data.get("isbn_10"):
@ -232,19 +238,30 @@ def ignore_edition(edition_data):
return True return True
def get_description(description_blob): def get_description(description_blob: Union[JsonDict, str]) -> str:
"""descriptions can be a string or a dict""" """descriptions can be a string or a dict"""
if isinstance(description_blob, dict): if isinstance(description_blob, dict):
return description_blob.get("value") description = markdown(description_blob.get("value", ""))
return description_blob else:
description = markdown(description_blob)
if (
description.startswith("<p>")
and description.endswith("</p>")
and description.count("<p>") == 1
):
# If there is just one <p> tag and it is around the text remove it
return description[len("<p>") : -len("</p>")].strip()
return clean(description)
def get_openlibrary_key(key): def get_openlibrary_key(key: str) -> str:
"""convert /books/OL27320736M into OL27320736M""" """convert /books/OL27320736M into OL27320736M"""
return key.split("/")[-1] return key.split("/")[-1]
def get_languages(language_blob): def get_languages(language_blob: Iterable[JsonDict]) -> list[Optional[str]]:
"""/language/eng -> English""" """/language/eng -> English"""
langs = [] langs = []
for lang in language_blob: for lang in language_blob:
@ -252,14 +269,14 @@ def get_languages(language_blob):
return langs return langs
def get_dict_field(blob, field_name): def get_dict_field(blob: Optional[JsonDict], field_name: str) -> Optional[Any]:
"""extract the isni from the remote id data for the author""" """extract the isni from the remote id data for the author"""
if not blob or not isinstance(blob, dict): if not blob or not isinstance(blob, dict):
return None return None
return blob.get(field_name) return blob.get(field_name)
def get_wikipedia_link(links): def get_wikipedia_link(links: list[Any]) -> Optional[str]:
"""extract wikipedia links""" """extract wikipedia links"""
if not isinstance(links, list): if not isinstance(links, list):
return None return None
@ -272,7 +289,7 @@ def get_wikipedia_link(links):
return None return None
def get_inventaire_id(links): def get_inventaire_id(links: list[Any]) -> Optional[str]:
"""extract and format inventaire ids""" """extract and format inventaire ids"""
if not isinstance(links, list): if not isinstance(links, list):
return None return None
@ -282,11 +299,13 @@ def get_inventaire_id(links):
continue continue
if link.get("title") == "inventaire.io": if link.get("title") == "inventaire.io":
iv_link = link.get("url") iv_link = link.get("url")
if not isinstance(iv_link, str):
return None
return iv_link.split("/")[-1] return iv_link.split("/")[-1]
return None return None
def pick_default_edition(options): def pick_default_edition(options: list[JsonDict]) -> Optional[JsonDict]:
"""favor physical copies with covers in english""" """favor physical copies with covers in english"""
if not options: if not options:
return None return None

View file

@ -3,7 +3,7 @@ from django.core.mail import EmailMultiAlternatives
from django.template.loader import get_template from django.template.loader import get_template
from bookwyrm import models, settings from bookwyrm import models, settings
from bookwyrm.tasks import app, HIGH from bookwyrm.tasks import app, EMAIL
from bookwyrm.settings import DOMAIN from bookwyrm.settings import DOMAIN
@ -18,6 +18,12 @@ def email_data():
} }
def test_email(user):
"""Just an admin checking if emails are sending"""
data = email_data()
send_email(user.email, *format_email("test", data))
def email_confirmation_email(user): def email_confirmation_email(user):
"""newly registered users confirm email address""" """newly registered users confirm email address"""
data = email_data() data = email_data()
@ -38,7 +44,7 @@ def password_reset_email(reset_code):
data = email_data() data = email_data()
data["reset_link"] = reset_code.link data["reset_link"] = reset_code.link
data["user"] = reset_code.user.display_name data["user"] = reset_code.user.display_name
send_email.delay(reset_code.user.email, *format_email("password_reset", data)) send_email(reset_code.user.email, *format_email("password_reset", data))
def moderation_report_email(report): def moderation_report_email(report):
@ -48,6 +54,7 @@ def moderation_report_email(report):
if report.user: if report.user:
data["reportee"] = report.user.localname or report.user.username data["reportee"] = report.user.localname or report.user.username
data["report_link"] = report.remote_id data["report_link"] = report.remote_id
data["link_domain"] = report.links.exists()
for admin in models.User.objects.filter( for admin in models.User.objects.filter(
groups__name__in=["admin", "moderator"] groups__name__in=["admin", "moderator"]
@ -68,7 +75,7 @@ def format_email(email_name, data):
return (subject, html_content, text_content) return (subject, html_content, text_content)
@app.task(queue=HIGH) @app.task(queue=EMAIL)
def send_email(recipient, subject, html_content, text_content): def send_email(recipient, subject, html_content, text_content):
"""use a task to send the email""" """use a task to send the email"""
email = EmailMultiAlternatives( email = EmailMultiAlternatives(

View file

@ -15,7 +15,7 @@ from .custom_form import CustomForm, StyledForm
# pylint: disable=missing-class-docstring # pylint: disable=missing-class-docstring
class ExpiryWidget(widgets.Select): class ExpiryWidget(widgets.Select):
def value_from_datadict(self, data, files, name): def value_from_datadict(self, data, files, name):
"""human-readable exiration time buckets""" """human-readable expiration time buckets"""
selected_string = super().value_from_datadict(data, files, name) selected_string = super().value_from_datadict(data, files, name)
if selected_string == "day": if selected_string == "day":
@ -55,11 +55,46 @@ class CreateInviteForm(CustomForm):
class SiteForm(CustomForm): class SiteForm(CustomForm):
class Meta: class Meta:
model = models.SiteSettings model = models.SiteSettings
exclude = ["admin_code", "install_mode"] fields = [
"name",
"instance_tagline",
"instance_description",
"instance_short_description",
"default_theme",
"code_of_conduct",
"privacy_policy",
"impressum",
"show_impressum",
"logo",
"logo_small",
"favicon",
"support_link",
"support_title",
"admin_email",
"footer_item",
]
widgets = { widgets = {
"instance_short_description": forms.TextInput( "instance_short_description": forms.TextInput(
attrs={"aria-describedby": "desc_instance_short_description"} attrs={"aria-describedby": "desc_instance_short_description"}
), ),
}
class RegistrationForm(CustomForm):
class Meta:
model = models.SiteSettings
fields = [
"allow_registration",
"allow_invite_requests",
"registration_closed_text",
"invite_request_text",
"invite_request_question",
"invite_question_text",
"require_confirm_email",
"default_user_auth_group",
]
widgets = {
"require_confirm_email": forms.CheckboxInput( "require_confirm_email": forms.CheckboxInput(
attrs={"aria-describedby": "desc_require_confirm_email"} attrs={"aria-describedby": "desc_require_confirm_email"}
), ),
@ -69,6 +104,23 @@ class SiteForm(CustomForm):
} }
class RegistrationLimitedForm(CustomForm):
class Meta:
model = models.SiteSettings
fields = [
"registration_closed_text",
"invite_request_text",
"invite_request_question",
"invite_question_text",
]
widgets = {
"invite_request_text": forms.Textarea(
attrs={"aria-describedby": "desc_invite_request_text"}
),
}
class ThemeForm(CustomForm): class ThemeForm(CustomForm):
class Meta: class Meta:
model = models.Theme model = models.Theme

View file

@ -15,12 +15,14 @@ class AuthorForm(CustomForm):
"aliases", "aliases",
"bio", "bio",
"wikipedia_link", "wikipedia_link",
"website",
"born", "born",
"died", "died",
"openlibrary_key", "openlibrary_key",
"inventaire_id", "inventaire_id",
"librarything_key", "librarything_key",
"goodreads_key", "goodreads_key",
"isfdb",
"isni", "isni",
] ]
widgets = { widgets = {
@ -30,10 +32,11 @@ class AuthorForm(CustomForm):
"wikipedia_link": forms.TextInput( "wikipedia_link": forms.TextInput(
attrs={"aria-describedby": "desc_wikipedia_link"} attrs={"aria-describedby": "desc_wikipedia_link"}
), ),
"website": forms.TextInput(attrs={"aria-describedby": "desc_website"}),
"born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}), "born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
"died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}), "died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),
"oepnlibrary_key": forms.TextInput( "openlibrary_key": forms.TextInput(
attrs={"aria-describedby": "desc_oepnlibrary_key"} attrs={"aria-describedby": "desc_openlibrary_key"}
), ),
"inventaire_id": forms.TextInput( "inventaire_id": forms.TextInput(
attrs={"aria-describedby": "desc_inventaire_id"} attrs={"aria-describedby": "desc_inventaire_id"}

View file

@ -18,22 +18,37 @@ class CoverForm(CustomForm):
class EditionForm(CustomForm): class EditionForm(CustomForm):
class Meta: class Meta:
model = models.Edition model = models.Edition
exclude = [ fields = [
"remote_id", "title",
"origin_id", "sort_title",
"created_date", "subtitle",
"updated_date", "description",
"edition_rank", "series",
"authors", "series_number",
"parent_work", "languages",
"shelves", "subjects",
"connector", "publishers",
"search_vector", "first_published_date",
"links", "published_date",
"file_links", "cover",
"physical_format",
"physical_format_detail",
"pages",
"isbn_13",
"isbn_10",
"openlibrary_key",
"inventaire_id",
"goodreads_key",
"oclc_number",
"asin",
"aasin",
"isfdb",
] ]
widgets = { widgets = {
"title": forms.TextInput(attrs={"aria-describedby": "desc_title"}), "title": forms.TextInput(attrs={"aria-describedby": "desc_title"}),
"sort_title": forms.TextInput(
attrs={"aria-describedby": "desc_sort_title"}
),
"subtitle": forms.TextInput(attrs={"aria-describedby": "desc_subtitle"}), "subtitle": forms.TextInput(attrs={"aria-describedby": "desc_subtitle"}),
"description": forms.Textarea( "description": forms.Textarea(
attrs={"aria-describedby": "desc_description"} attrs={"aria-describedby": "desc_description"}
@ -73,10 +88,15 @@ class EditionForm(CustomForm):
"inventaire_id": forms.TextInput( "inventaire_id": forms.TextInput(
attrs={"aria-describedby": "desc_inventaire_id"} attrs={"aria-describedby": "desc_inventaire_id"}
), ),
"goodreads_key": forms.TextInput(
attrs={"aria-describedby": "desc_goodreads_key"}
),
"oclc_number": forms.TextInput( "oclc_number": forms.TextInput(
attrs={"aria-describedby": "desc_oclc_number"} attrs={"aria-describedby": "desc_oclc_number"}
), ),
"ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}), "ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}),
"AASIN": forms.TextInput(attrs={"aria-describedby": "desc_AASIN"}),
"isfdb": forms.TextInput(attrs={"aria-describedby": "desc_isfdb"}),
} }
@ -91,6 +111,7 @@ class EditionFromWorkForm(CustomForm):
model = models.Work model = models.Work
fields = [ fields = [
"title", "title",
"sort_title",
"subtitle", "subtitle",
"authors", "authors",
"description", "description",

View file

@ -8,6 +8,7 @@ import pyotp
from bookwyrm import models from bookwyrm import models
from bookwyrm.settings import DOMAIN from bookwyrm.settings import DOMAIN
from bookwyrm.settings import TWO_FACTOR_LOGIN_VALIDITY_WINDOW
from .custom_form import CustomForm from .custom_form import CustomForm
@ -108,7 +109,7 @@ class Confirm2FAForm(CustomForm):
otp = self.data.get("otp") otp = self.data.get("otp")
totp = pyotp.TOTP(self.instance.otp_secret) totp = pyotp.TOTP(self.instance.otp_secret)
if not totp.verify(otp): if not totp.verify(otp, valid_window=TWO_FACTOR_LOGIN_VALIDITY_WINDOW):
if self.instance.hotp_secret: if self.instance.hotp_secret:
# maybe it's a backup code? # maybe it's a backup code?

View file

@ -36,13 +36,16 @@ class FileLinkForm(CustomForm):
"This domain is blocked. Please contact your administrator if you think this is an error." "This domain is blocked. Please contact your administrator if you think this is an error."
), ),
) )
elif models.FileLink.objects.filter( if (
not self.instance
and models.FileLink.objects.filter(
url=url, book=book, filetype=filetype url=url, book=book, filetype=filetype
).exists(): ).exists()
# pylint: disable=line-too-long ):
self.add_error( # pylint: disable=line-too-long
"url", self.add_error(
_( "url",
"This link with file type has already been added for this book. If it is not visible, the domain is still pending." _(
), "This link with file type has already been added for this book. If it is not visible, the domain is still pending."
) ),
)

View file

@ -24,7 +24,7 @@ class SortListForm(forms.Form):
sort_by = ChoiceField( sort_by = ChoiceField(
choices=( choices=(
("order", _("List Order")), ("order", _("List Order")),
("title", _("Book Title")), ("sort_title", _("Book Title")),
("rating", _("Rating")), ("rating", _("Rating")),
), ),
label=_("Sort By"), label=_("Sort By"),

View file

@ -53,6 +53,7 @@ class QuotationForm(CustomForm):
"sensitive", "sensitive",
"privacy", "privacy",
"position", "position",
"endposition",
"position_mode", "position_mode",
] ]

View file

@ -1,4 +1,6 @@
""" handle reading a csv from calibre """ """ handle reading a csv from calibre """
from typing import Any, Optional
from bookwyrm.models import Shelf from bookwyrm.models import Shelf
from . import Importer from . import Importer
@ -9,7 +11,7 @@ class CalibreImporter(Importer):
service = "Calibre" service = "Calibre"
def __init__(self, *args, **kwargs): def __init__(self, *args: Any, **kwargs: Any):
# Add timestamp to row_mappings_guesses for date_added to avoid # Add timestamp to row_mappings_guesses for date_added to avoid
# integrity error # integrity error
row_mappings_guesses = [] row_mappings_guesses = []
@ -23,6 +25,6 @@ class CalibreImporter(Importer):
self.row_mappings_guesses = row_mappings_guesses self.row_mappings_guesses = row_mappings_guesses
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def get_shelf(self, normalized_row): def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
# Calibre export does not indicate which shelf to use. Use a default one for now # Calibre export does not indicate which shelf to use. Use a default one for now
return Shelf.TO_READ return Shelf.TO_READ

View file

@ -1,7 +1,10 @@
""" handle reading a csv from an external service, defaults are from Goodreads """ """ handle reading a csv from an external service, defaults are from Goodreads """
import csv import csv
from datetime import timedelta
from typing import Iterable, Optional
from django.utils import timezone from django.utils import timezone
from bookwyrm.models import ImportJob, ImportItem from bookwyrm.models import ImportJob, ImportItem, SiteSettings, User
class Importer: class Importer:
@ -16,8 +19,8 @@ class Importer:
("id", ["id", "book id"]), ("id", ["id", "book id"]),
("title", ["title"]), ("title", ["title"]),
("authors", ["author", "authors", "primary author"]), ("authors", ["author", "authors", "primary author"]),
("isbn_10", ["isbn10", "isbn"]), ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
("isbn_13", ["isbn13", "isbn", "isbns"]), ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]), ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
("review_name", ["review name"]), ("review_name", ["review name"]),
("review_body", ["my review", "review"]), ("review_body", ["my review", "review"]),
@ -33,26 +36,48 @@ class Importer:
"reading": ["currently-reading", "reading", "currently reading"], "reading": ["currently-reading", "reading", "currently reading"],
} }
def create_job(self, user, csv_file, include_reviews, privacy): # pylint: disable=too-many-locals
def create_job(
self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str
) -> ImportJob:
"""check over a csv and creates a database entry for the job""" """check over a csv and creates a database entry for the job"""
csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter) csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
rows = enumerate(list(csv_reader)) rows = list(csv_reader)
if len(rows) < 1:
raise ValueError("CSV file is empty")
mappings = (
self.create_row_mappings(list(fieldnames))
if (fieldnames := csv_reader.fieldnames)
else {}
)
job = ImportJob.objects.create( job = ImportJob.objects.create(
user=user, user=user,
include_reviews=include_reviews, include_reviews=include_reviews,
privacy=privacy, privacy=privacy,
mappings=self.create_row_mappings(csv_reader.fieldnames), mappings=mappings,
source=self.service, source=self.service,
) )
for index, entry in rows: enforce_limit, allowed_imports = self.get_import_limit(user)
if enforce_limit and allowed_imports <= 0:
job.complete_job()
return job
for index, entry in enumerate(rows):
if enforce_limit and index >= allowed_imports:
break
self.create_item(job, index, entry) self.create_item(job, index, entry)
return job return job
def update_legacy_job(self, job): def update_legacy_job(self, job: ImportJob) -> None:
"""patch up a job that was in the old format""" """patch up a job that was in the old format"""
items = job.items items = job.items
headers = list(items.first().data.keys()) first_item = items.first()
if first_item is None:
return
headers = list(first_item.data.keys())
job.mappings = self.create_row_mappings(headers) job.mappings = self.create_row_mappings(headers)
job.updated_date = timezone.now() job.updated_date = timezone.now()
job.save() job.save()
@ -63,24 +88,24 @@ class Importer:
item.normalized_data = normalized item.normalized_data = normalized
item.save() item.save()
def create_row_mappings(self, headers): def create_row_mappings(self, headers: list[str]) -> dict[str, Optional[str]]:
"""guess what the headers mean""" """guess what the headers mean"""
mappings = {} mappings = {}
for (key, guesses) in self.row_mappings_guesses: for (key, guesses) in self.row_mappings_guesses:
value = [h for h in headers if h.lower() in guesses] values = [h for h in headers if h.lower() in guesses]
value = value[0] if len(value) else None value = values[0] if len(values) else None
if value: if value:
headers.remove(value) headers.remove(value)
mappings[key] = value mappings[key] = value
return mappings return mappings
def create_item(self, job, index, data): def create_item(self, job: ImportJob, index: int, data: dict[str, str]) -> None:
"""creates and saves an import item""" """creates and saves an import item"""
normalized = self.normalize_row(data, job.mappings) normalized = self.normalize_row(data, job.mappings)
normalized["shelf"] = self.get_shelf(normalized) normalized["shelf"] = self.get_shelf(normalized)
ImportItem(job=job, index=index, data=data, normalized_data=normalized).save() ImportItem(job=job, index=index, data=data, normalized_data=normalized).save()
def get_shelf(self, normalized_row): def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
"""determine which shelf to use""" """determine which shelf to use"""
shelf_name = normalized_row.get("shelf") shelf_name = normalized_row.get("shelf")
if not shelf_name: if not shelf_name:
@ -91,11 +116,35 @@ class Importer:
] ]
return shelf[0] if shelf else None return shelf[0] if shelf else None
def normalize_row(self, entry, mappings): # pylint: disable=no-self-use # pylint: disable=no-self-use
def normalize_row(
self, entry: dict[str, str], mappings: dict[str, Optional[str]]
) -> dict[str, Optional[str]]:
"""use the dataclass to create the formatted row of data""" """use the dataclass to create the formatted row of data"""
return {k: entry.get(v) for k, v in mappings.items()} return {k: entry.get(v) if v else None for k, v in mappings.items()}
def create_retry_job(self, user, original_job, items): # pylint: disable=no-self-use
def get_import_limit(self, user: User) -> tuple[int, int]:
"""check if import limit is set and return how many imports are left"""
site_settings = SiteSettings.objects.get()
import_size_limit = site_settings.import_size_limit
import_limit_reset = site_settings.import_limit_reset
enforce_limit = import_size_limit and import_limit_reset
allowed_imports = 0
if enforce_limit:
time_range = timezone.now() - timedelta(days=import_limit_reset)
import_jobs = ImportJob.objects.filter(
user=user, created_date__gte=time_range
)
# pylint: disable=consider-using-generator
imported_books = sum([job.successful_item_count for job in import_jobs])
allowed_imports = import_size_limit - imported_books
return enforce_limit, allowed_imports
def create_retry_job(
self, user: User, original_job: ImportJob, items: list[ImportItem]
) -> ImportJob:
"""retry items that didn't import""" """retry items that didn't import"""
job = ImportJob.objects.create( job = ImportJob.objects.create(
user=user, user=user,
@ -106,7 +155,13 @@ class Importer:
mappings=original_job.mappings, mappings=original_job.mappings,
retry=True, retry=True,
) )
for item in items: enforce_limit, allowed_imports = self.get_import_limit(user)
if enforce_limit and allowed_imports <= 0:
job.complete_job()
return job
for index, item in enumerate(items):
if enforce_limit and index >= allowed_imports:
break
# this will re-normalize the raw data # this will re-normalize the raw data
self.create_item(job, item.index, item.data) self.create_item(job, item.index, item.data)
return job return job

View file

@ -1,11 +1,16 @@
""" handle reading a tsv from librarything """ """ handle reading a tsv from librarything """
import re import re
from typing import Optional
from bookwyrm.models import Shelf from bookwyrm.models import Shelf
from . import Importer from . import Importer
def _remove_brackets(value: Optional[str]) -> Optional[str]:
return re.sub(r"\[|\]", "", value) if value else None
class LibrarythingImporter(Importer): class LibrarythingImporter(Importer):
"""csv downloads from librarything""" """csv downloads from librarything"""
@ -13,16 +18,19 @@ class LibrarythingImporter(Importer):
delimiter = "\t" delimiter = "\t"
encoding = "ISO-8859-1" encoding = "ISO-8859-1"
def normalize_row(self, entry, mappings): # pylint: disable=no-self-use def normalize_row(
self, entry: dict[str, str], mappings: dict[str, Optional[str]]
) -> dict[str, Optional[str]]: # pylint: disable=no-self-use
"""use the dataclass to create the formatted row of data""" """use the dataclass to create the formatted row of data"""
remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None normalized = {
normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()} k: _remove_brackets(entry.get(v) if v else None)
isbn_13 = normalized.get("isbn_13") for k, v in mappings.items()
isbn_13 = isbn_13.split(", ") if isbn_13 else [] }
normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 0 else None isbn_13 = value.split(", ") if (value := normalized.get("isbn_13")) else []
normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 1 else None
return normalized return normalized
def get_shelf(self, normalized_row): def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
if normalized_row["date_finished"]: if normalized_row["date_finished"]:
return Shelf.READ_FINISHED return Shelf.READ_FINISHED
if normalized_row["date_started"]: if normalized_row["date_started"]:

View file

@ -1,4 +1,6 @@
""" handle reading a csv from openlibrary""" """ handle reading a csv from openlibrary"""
from typing import Any
from . import Importer from . import Importer
@ -7,7 +9,7 @@ class OpenLibraryImporter(Importer):
service = "OpenLibrary" service = "OpenLibrary"
def __init__(self, *args, **kwargs): def __init__(self, *args: Any, **kwargs: Any):
self.row_mappings_guesses.append(("openlibrary_key", ["edition id"])) self.row_mappings_guesses.append(("openlibrary_key", ["edition id"]))
self.row_mappings_guesses.append(("openlibrary_work_key", ["work id"])) self.row_mappings_guesses.append(("openlibrary_work_key", ["work id"]))
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)

File diff suppressed because it is too large Load diff

View file

123
bookwyrm/isbn/isbn.py Normal file
View file

@ -0,0 +1,123 @@
""" Use the range message from isbn-international to hyphenate ISBNs """
import os
from typing import Optional
from xml.etree import ElementTree
from xml.etree.ElementTree import Element
import requests
from bookwyrm import settings
def _get_rules(element: Element) -> list[Element]:
if (rules_el := element.find("Rules")) is not None:
return rules_el.findall("Rule")
return []
class IsbnHyphenator:
"""Class to manage the range message xml file and use it to hyphenate ISBNs"""
__range_message_url = "https://www.isbn-international.org/export_rangemessage.xml"
__range_file_path = os.path.join(
settings.BASE_DIR, "bookwyrm", "isbn", "RangeMessage.xml"
)
__element_tree = None
def update_range_message(self) -> None:
"""Download the range message xml file and save it locally"""
response = requests.get(self.__range_message_url)
with open(self.__range_file_path, "w", encoding="utf-8") as file:
file.write(response.text)
self.__element_tree = None
def hyphenate(self, isbn_13: Optional[str]) -> Optional[str]:
"""hyphenate the given ISBN-13 number using the range message"""
if isbn_13 is None:
return None
if self.__element_tree is None:
self.__element_tree = ElementTree.parse(self.__range_file_path)
gs1_prefix = isbn_13[:3]
reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
if reg_group is None:
return isbn_13 # failed to hyphenate
registrant = self.__find_registrant(isbn_13, gs1_prefix, reg_group)
if registrant is None:
return isbn_13 # failed to hyphenate
publication = isbn_13[len(gs1_prefix) + len(reg_group) + len(registrant) : -1]
check_digit = isbn_13[-1:]
return "-".join((gs1_prefix, reg_group, registrant, publication, check_digit))
def __find_reg_group(self, isbn_13: str, gs1_prefix: str) -> Optional[str]:
if self.__element_tree is None:
self.__element_tree = ElementTree.parse(self.__range_file_path)
ucc_prefixes_el = self.__element_tree.find("EAN.UCCPrefixes")
if ucc_prefixes_el is None:
return None
for ean_ucc_el in ucc_prefixes_el.findall("EAN.UCC"):
if (
prefix_el := ean_ucc_el.find("Prefix")
) is not None and prefix_el.text == gs1_prefix:
for rule_el in _get_rules(ean_ucc_el):
length_el = rule_el.find("Length")
if length_el is None:
continue
length = int(text) if (text := length_el.text) else 0
if length == 0:
continue
range_el = rule_el.find("Range")
if range_el is None or range_el.text is None:
continue
reg_grp_range = [int(x[:length]) for x in range_el.text.split("-")]
reg_group = isbn_13[len(gs1_prefix) : len(gs1_prefix) + length]
if reg_grp_range[0] <= int(reg_group) <= reg_grp_range[1]:
return reg_group
return None
return None
def __find_registrant(
self, isbn_13: str, gs1_prefix: str, reg_group: str
) -> Optional[str]:
from_ind = len(gs1_prefix) + len(reg_group)
if self.__element_tree is None:
self.__element_tree = ElementTree.parse(self.__range_file_path)
reg_groups_el = self.__element_tree.find("RegistrationGroups")
if reg_groups_el is None:
return None
for group_el in reg_groups_el.findall("Group"):
if (
prefix_el := group_el.find("Prefix")
) is not None and prefix_el.text == "-".join((gs1_prefix, reg_group)):
for rule_el in _get_rules(group_el):
length_el = rule_el.find("Length")
if length_el is None:
continue
length = int(text) if (text := length_el.text) else 0
if length == 0:
continue
range_el = rule_el.find("Range")
if range_el is None or range_el.text is None:
continue
registrant_range = [
int(x[:length]) for x in range_el.text.split("-")
]
registrant = isbn_13[from_ind : from_ind + length]
if registrant_range[0] <= int(registrant) <= registrant_range[1]:
return registrant
return None
return None
hyphenator_singleton = IsbnHyphenator()

View file

@ -5,7 +5,7 @@ from django.db.models import signals, Count, Q
from bookwyrm import models from bookwyrm import models
from bookwyrm.redis_store import RedisStore from bookwyrm.redis_store import RedisStore
from bookwyrm.tasks import app, MEDIUM, HIGH from bookwyrm.tasks import app, LISTS
class ListsStream(RedisStore): class ListsStream(RedisStore):
@ -24,8 +24,7 @@ class ListsStream(RedisStore):
def add_list(self, book_list): def add_list(self, book_list):
"""add a list to users' feeds""" """add a list to users' feeds"""
# the pipeline contains all the add-to-stream activities self.add_object_to_stores(book_list, self.get_stores_for_object(book_list))
self.add_object_to_related_stores(book_list)
def add_user_lists(self, viewer, user): def add_user_lists(self, viewer, user):
"""add a user's lists to another user's feed""" """add a user's lists to another user's feed"""
@ -86,18 +85,19 @@ class ListsStream(RedisStore):
if group: if group:
audience = audience.filter( audience = audience.filter(
Q(id=book_list.user.id) # if the user is the list's owner Q(id=book_list.user.id) # if the user is the list's owner
| Q(following=book_list.user) # if the user is following the pwmer | Q(following=book_list.user) # if the user is following the owner
# if a user is in the group # if a user is in the group
| Q(memberships__group__id=book_list.group.id) | Q(memberships__group__id=book_list.group.id)
) )
else: else:
audience = audience.filter( audience = audience.filter(
Q(id=book_list.user.id) # if the user is the list's owner Q(id=book_list.user.id) # if the user is the list's owner
| Q(following=book_list.user) # if the user is following the pwmer | Q(following=book_list.user) # if the user is following the owner
) )
return audience.distinct() return audience.distinct()
def get_stores_for_object(self, obj): def get_stores_for_object(self, obj):
"""the stores that an object belongs in"""
return [self.stream_id(u) for u in self.get_audience(obj)] return [self.stream_id(u) for u in self.get_audience(obj)]
def get_lists_for_user(self, user): # pylint: disable=no-self-use def get_lists_for_user(self, user): # pylint: disable=no-self-use
@ -217,14 +217,14 @@ def add_list_on_account_create_command(user_id):
# ---- TASKS # ---- TASKS
@app.task(queue=MEDIUM) @app.task(queue=LISTS)
def populate_lists_task(user_id): def populate_lists_task(user_id):
"""background task for populating an empty list stream""" """background task for populating an empty list stream"""
user = models.User.objects.get(id=user_id) user = models.User.objects.get(id=user_id)
ListsStream().populate_lists(user) ListsStream().populate_lists(user)
@app.task(queue=MEDIUM) @app.task(queue=LISTS)
def remove_list_task(list_id, re_add=False): def remove_list_task(list_id, re_add=False):
"""remove a list from any stream it might be in""" """remove a list from any stream it might be in"""
stores = models.User.objects.filter(local=True, is_active=True).values_list( stores = models.User.objects.filter(local=True, is_active=True).values_list(
@ -233,20 +233,20 @@ def remove_list_task(list_id, re_add=False):
# delete for every store # delete for every store
stores = [ListsStream().stream_id(idx) for idx in stores] stores = [ListsStream().stream_id(idx) for idx in stores]
ListsStream().remove_object_from_related_stores(list_id, stores=stores) ListsStream().remove_object_from_stores(list_id, stores)
if re_add: if re_add:
add_list_task.delay(list_id) add_list_task.delay(list_id)
@app.task(queue=HIGH) @app.task(queue=LISTS)
def add_list_task(list_id): def add_list_task(list_id):
"""add a list to any stream it should be in""" """add a list to any stream it should be in"""
book_list = models.List.objects.get(id=list_id) book_list = models.List.objects.get(id=list_id)
ListsStream().add_list(book_list) ListsStream().add_list(book_list)
@app.task(queue=MEDIUM) @app.task(queue=LISTS)
def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None): def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
"""remove all lists by a user from a viewer's stream""" """remove all lists by a user from a viewer's stream"""
viewer = models.User.objects.get(id=viewer_id) viewer = models.User.objects.get(id=viewer_id)
@ -254,7 +254,7 @@ def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy) ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy)
@app.task(queue=MEDIUM) @app.task(queue=LISTS)
def add_user_lists_task(viewer_id, user_id): def add_user_lists_task(viewer_id, user_id):
"""add all lists by a user to a viewer's stream""" """add all lists by a user to a viewer's stream"""
viewer = models.User.objects.get(id=viewer_id) viewer = models.User.objects.get(id=viewer_id)

View file

@ -0,0 +1,48 @@
""" Our own command to all scss themes """
import glob
import os
import sass
from django.core.management.base import BaseCommand
from sass_processor.apps import APPS_INCLUDE_DIRS
from sass_processor.processor import SassProcessor
from sass_processor.utils import get_custom_functions
from bookwyrm import settings
class Command(BaseCommand):
"""command-line options"""
help = "SCSS compile all BookWyrm themes"
# pylint: disable=unused-argument
def handle(self, *args, **options):
"""compile"""
themes_dir = os.path.join(
settings.BASE_DIR, "bookwyrm", "static", "css", "themes", "*.scss"
)
for theme_scss in glob.glob(themes_dir):
basename, _ = os.path.splitext(theme_scss)
theme_css = f"{basename}.css"
self.compile_sass(theme_scss, theme_css)
def compile_sass(self, sass_path, css_path):
compile_kwargs = {
"filename": sass_path,
"include_paths": SassProcessor.include_paths + APPS_INCLUDE_DIRS,
"custom_functions": get_custom_functions(),
"precision": getattr(settings, "SASS_PRECISION", 8),
"output_style": getattr(
settings,
"SASS_OUTPUT_STYLE",
"nested" if settings.DEBUG else "compressed",
),
}
content = sass.compile(**compile_kwargs)
with open(css_path, "w") as f:
f.write(content)
self.stdout.write("Compiled SASS/SCSS file: '{0}'\n".format(sass_path))

View file

@ -0,0 +1,19 @@
""" manually confirm e-mail of user """
from django.core.management.base import BaseCommand
from bookwyrm import models
class Command(BaseCommand):
"""command-line options"""
help = "Manually confirm email for user"
def add_arguments(self, parser):
parser.add_argument("username")
def handle(self, *args, **options):
name = options["username"]
user = models.User.objects.get(localname=name)
user.reactivate()
self.stdout.write(self.style.SUCCESS("User's email is now confirmed."))

View file

@ -3,38 +3,7 @@ merge book data objects """
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.db.models import Count from django.db.models import Count
from bookwyrm import models from bookwyrm import models
from bookwyrm.management.merge import merge_objects
def update_related(canonical, obj):
"""update all the models with fk to the object being removed"""
# move related models to canonical
related_models = [
(r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
]
for (related_field, related_model) in related_models:
related_objs = related_model.objects.filter(**{related_field: obj})
for related_obj in related_objs:
print("replacing in", related_model.__name__, related_field, related_obj.id)
try:
setattr(related_obj, related_field, canonical)
related_obj.save()
except TypeError:
getattr(related_obj, related_field).add(canonical)
getattr(related_obj, related_field).remove(obj)
def copy_data(canonical, obj):
"""try to get the most data possible"""
for data_field in obj._meta.get_fields():
if not hasattr(data_field, "activitypub_field"):
continue
data_value = getattr(obj, data_field.name)
if not data_value:
continue
if not getattr(canonical, data_field.name):
print("setting data field", data_field.name, data_value)
setattr(canonical, data_field.name, data_value)
canonical.save()
def dedupe_model(model): def dedupe_model(model):
@ -61,19 +30,16 @@ def dedupe_model(model):
print("keeping", canonical.remote_id) print("keeping", canonical.remote_id)
for obj in objs[1:]: for obj in objs[1:]:
print(obj.remote_id) print(obj.remote_id)
copy_data(canonical, obj) merge_objects(canonical, obj)
update_related(canonical, obj)
# remove the outdated entry
obj.delete()
class Command(BaseCommand): class Command(BaseCommand):
"""dedplucate allllll the book data models""" """deduplicate allllll the book data models"""
help = "merges duplicate book data" help = "merges duplicate book data"
# pylint: disable=no-self-use,unused-argument # pylint: disable=no-self-use,unused-argument
def handle(self, *args, **options): def handle(self, *args, **options):
"""run deudplications""" """run deduplications"""
dedupe_model(models.Edition) dedupe_model(models.Edition)
dedupe_model(models.Work) dedupe_model(models.Work)
dedupe_model(models.Author) dedupe_model(models.Author)

View file

@ -4,12 +4,7 @@ import redis
from bookwyrm import settings from bookwyrm import settings
r = redis.Redis( r = redis.from_url(settings.REDIS_ACTIVITY_URL)
host=settings.REDIS_ACTIVITY_HOST,
port=settings.REDIS_ACTIVITY_PORT,
password=settings.REDIS_ACTIVITY_PASSWORD,
db=settings.REDIS_ACTIVITY_DB_INDEX,
)
def erase_streams(): def erase_streams():

View file

@ -8,54 +8,64 @@ from bookwyrm import models
def init_groups(): def init_groups():
"""permission levels""" """permission levels"""
groups = ["admin", "moderator", "editor"] groups = ["admin", "owner", "moderator", "editor"]
for group in groups: for group in groups:
Group.objects.create(name=group) Group.objects.get_or_create(name=group)
def init_permissions(): def init_permissions():
"""permission types""" """permission types"""
permissions = [ permissions = [
{
"codename": "manage_registration",
"name": "allow or prevent user registration",
"groups": ["admin"],
},
{
"codename": "system_administration",
"name": "technical controls",
"groups": ["admin"],
},
{ {
"codename": "edit_instance_settings", "codename": "edit_instance_settings",
"name": "change the instance info", "name": "change the instance info",
"groups": ["admin"], "groups": ["admin", "owner"],
}, },
{ {
"codename": "set_user_group", "codename": "set_user_group",
"name": "change what group a user is in", "name": "change what group a user is in",
"groups": ["admin", "moderator"], "groups": ["admin", "owner", "moderator"],
}, },
{ {
"codename": "control_federation", "codename": "control_federation",
"name": "control who to federate with", "name": "control who to federate with",
"groups": ["admin", "moderator"], "groups": ["admin", "owner", "moderator"],
}, },
{ {
"codename": "create_invites", "codename": "create_invites",
"name": "issue invitations to join", "name": "issue invitations to join",
"groups": ["admin", "moderator"], "groups": ["admin", "owner", "moderator"],
}, },
{ {
"codename": "moderate_user", "codename": "moderate_user",
"name": "deactivate or silence a user", "name": "deactivate or silence a user",
"groups": ["admin", "moderator"], "groups": ["admin", "owner", "moderator"],
}, },
{ {
"codename": "moderate_post", "codename": "moderate_post",
"name": "delete other users' posts", "name": "delete other users' posts",
"groups": ["admin", "moderator"], "groups": ["admin", "owner", "moderator"],
}, },
{ {
"codename": "edit_book", "codename": "edit_book",
"name": "edit book info", "name": "edit book info",
"groups": ["admin", "moderator", "editor"], "groups": ["admin", "owner", "moderator", "editor"],
}, },
] ]
content_type = ContentType.objects.get_for_model(models.User) content_type = ContentType.objects.get_for_model(models.User)
for permission in permissions: for permission in permissions:
permission_obj = Permission.objects.create( permission_obj, _ = Permission.objects.get_or_create(
codename=permission["codename"], codename=permission["codename"],
name=permission["name"], name=permission["name"],
content_type=content_type, content_type=content_type,
@ -107,10 +117,12 @@ def init_connectors():
def init_settings(): def init_settings():
"""info about the instance""" """info about the instance"""
group_editor = Group.objects.filter(name="editor").first()
models.SiteSettings.objects.create( models.SiteSettings.objects.create(
support_link="https://www.patreon.com/bookwyrm", support_link="https://www.patreon.com/bookwyrm",
support_title="Patreon", support_title="Patreon",
install_mode=True, install_mode=True,
default_user_auth_group=group_editor,
) )

View file

@ -0,0 +1,12 @@
""" PROCEED WITH CAUTION: uses deduplication fields to permanently
merge author data objects """
from bookwyrm import models
from bookwyrm.management.merge_command import MergeCommand
class Command(MergeCommand):
"""merges two authors by ID"""
help = "merges specified authors into one"
MODEL = models.Author

View file

@ -0,0 +1,12 @@
""" PROCEED WITH CAUTION: uses deduplication fields to permanently
merge edition data objects """
from bookwyrm import models
from bookwyrm.management.merge_command import MergeCommand
class Command(MergeCommand):
"""merges two editions by ID"""
help = "merges specified editions into one"
MODEL = models.Edition

View file

@ -0,0 +1,12 @@
""" PROCEED WITH CAUTION: uses deduplication fields to permanently
merge work data objects """
from bookwyrm import models
from bookwyrm.management.merge_command import MergeCommand
class Command(MergeCommand):
"""merges two works by ID"""
help = "merges specified works into one"
MODEL = models.Work

View file

@ -0,0 +1,22 @@
"""deactivate two factor auth"""
from django.core.management.base import BaseCommand, CommandError
from bookwyrm import models
class Command(BaseCommand):
"""command-line options"""
help = "Remove Two Factor Authorisation from user"
def add_arguments(self, parser):
parser.add_argument("username")
def handle(self, *args, **options):
name = options["username"]
user = models.User.objects.get(localname=name)
user.two_factor_auth = False
user.save(broadcast=False, update_fields=["two_factor_auth"])
self.stdout.write(
self.style.SUCCESS("Two Factor Authorisation was removed from user")
)

View file

@ -33,10 +33,10 @@ def remove_editions():
class Command(BaseCommand): class Command(BaseCommand):
"""dedplucate allllll the book data models""" """deduplicate allllll the book data models"""
help = "merges duplicate book data" help = "merges duplicate book data"
# pylint: disable=no-self-use,unused-argument # pylint: disable=no-self-use,unused-argument
def handle(self, *args, **options): def handle(self, *args, **options):
"""run deudplications""" """run deduplications"""
remove_editions() remove_editions()

View file

@ -0,0 +1,40 @@
""" Remove preview images for remote users """
from django.core.management.base import BaseCommand
from django.db.models import Q
from bookwyrm import models, preview_images
# pylint: disable=line-too-long
class Command(BaseCommand):
"""Remove preview images for remote users"""
help = "Remove preview images for remote users"
# pylint: disable=no-self-use,unused-argument
def handle(self, *args, **options):
"""generate preview images"""
self.stdout.write(
" | Hello! I will be removing preview images from remote users."
)
self.stdout.write(
"🧑‍🚒 ⎨ This might take quite long if your instance has a lot of remote users."
)
self.stdout.write(" | ✧ Thank you for your patience ✧")
users = models.User.objects.filter(local=False).exclude(
Q(preview_image="") | Q(preview_image=None)
)
if len(users) > 0:
self.stdout.write(
f" → Remote user preview images ({len(users)}): ", ending=""
)
for user in users:
preview_images.remove_user_preview_image_task.delay(user.id)
self.stdout.write(".", ending="")
self.stdout.write(" OK 🖼")
else:
self.stdout.write(f" | There was no remote users with preview images.")
self.stdout.write("🧑‍🚒 ⎨ Im all done! ✧ Enjoy ✧")

View file

@ -0,0 +1,21 @@
""" Repair editions with missing works """
from django.core.management.base import BaseCommand
from bookwyrm import models
class Command(BaseCommand):
"""command-line options"""
help = "Repairs an edition that is in a broken state"
# pylint: disable=unused-argument
def handle(self, *args, **options):
"""Find and repair broken editions"""
# Find broken editions
editions = models.Edition.objects.filter(parent_work__isnull=True)
self.stdout.write(f"Repairing {editions.count()} edition(s):")
# Do repair
for edition in editions:
edition.repair()
self.stdout.write(".", ending="")

View file

@ -0,0 +1,31 @@
""" Actually let's not generate those preview images """
import json
from django.core.management.base import BaseCommand
from bookwyrm.tasks import app
class Command(BaseCommand):
"""Find and revoke image tasks"""
# pylint: disable=unused-argument
def handle(self, *args, **options):
"""revoke nonessential low priority tasks"""
types = [
"bookwyrm.preview_images.generate_edition_preview_image_task",
"bookwyrm.preview_images.generate_user_preview_image_task",
]
self.stdout.write(" | Finding tasks of types:")
self.stdout.write("\n".join(types))
with app.pool.acquire(block=True) as conn:
tasks = conn.default_channel.client.lrange("low_priority", 0, -1)
self.stdout.write(f" | Found {len(tasks)} task(s) in low priority queue")
revoke_ids = []
for task in tasks:
task_json = json.loads(task)
task_type = task_json.get("headers", {}).get("task")
if task_type in types:
revoke_ids.append(task_json.get("headers", {}).get("id"))
self.stdout.write(".", ending="")
self.stdout.write(f"\n | Revoking {len(revoke_ids)} task(s)")
app.control.revoke(revoke_ids)

View file

@ -0,0 +1,50 @@
from django.db.models import ManyToManyField
def update_related(canonical, obj):
"""update all the models with fk to the object being removed"""
# move related models to canonical
related_models = [
(r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
]
for (related_field, related_model) in related_models:
# Skip the ManyToMany fields that arent auto-created. These
# should have a corresponding OneToMany field in the model for
# the linking table anyway. If we update it through that model
# instead then we wont lose the extra fields in the linking
# table.
related_field_obj = related_model._meta.get_field(related_field)
if isinstance(related_field_obj, ManyToManyField):
through = related_field_obj.remote_field.through
if not through._meta.auto_created:
continue
related_objs = related_model.objects.filter(**{related_field: obj})
for related_obj in related_objs:
print("replacing in", related_model.__name__, related_field, related_obj.id)
try:
setattr(related_obj, related_field, canonical)
related_obj.save()
except TypeError:
getattr(related_obj, related_field).add(canonical)
getattr(related_obj, related_field).remove(obj)
def copy_data(canonical, obj):
"""try to get the most data possible"""
for data_field in obj._meta.get_fields():
if not hasattr(data_field, "activitypub_field"):
continue
data_value = getattr(obj, data_field.name)
if not data_value:
continue
if not getattr(canonical, data_field.name):
print("setting data field", data_field.name, data_value)
setattr(canonical, data_field.name, data_value)
canonical.save()
def merge_objects(canonical, obj):
copy_data(canonical, obj)
update_related(canonical, obj)
# remove the outdated entry
obj.delete()

View file

@ -0,0 +1,29 @@
from bookwyrm.management.merge import merge_objects
from django.core.management.base import BaseCommand
class MergeCommand(BaseCommand):
"""base class for merge commands"""
def add_arguments(self, parser):
"""add the arguments for this command"""
parser.add_argument("--canonical", type=int, required=True)
parser.add_argument("--other", type=int, required=True)
# pylint: disable=no-self-use,unused-argument
def handle(self, *args, **options):
"""merge the two objects"""
model = self.MODEL
try:
canonical = model.objects.get(id=options["canonical"])
except model.DoesNotExist:
print("canonical book doesnt exist!")
return
try:
other = model.objects.get(id=options["other"])
except model.DoesNotExist:
print("other book doesnt exist!")
return
merge_objects(canonical, other)

View file

@ -1467,7 +1467,7 @@ class Migration(migrations.Migration):
( (
"expiry", "expiry",
models.DateTimeField( models.DateTimeField(
default=bookwyrm.models.site.get_passowrd_reset_expiry default=bookwyrm.models.site.get_password_reset_expiry
), ),
), ),
( (

View file

@ -6,7 +6,7 @@ from bookwyrm.connectors.abstract_connector import infer_physical_format
def infer_format(app_registry, schema_editor): def infer_format(app_registry, schema_editor):
"""set the new phsyical format field based on existing format data""" """set the new physical format field based on existing format data"""
db_alias = schema_editor.connection.alias db_alias = schema_editor.connection.alias
editions = ( editions = (

View file

@ -5,7 +5,7 @@ from bookwyrm.settings import DOMAIN
def remove_self_connector(app_registry, schema_editor): def remove_self_connector(app_registry, schema_editor):
"""set the new phsyical format field based on existing format data""" """set the new physical format field based on existing format data"""
db_alias = schema_editor.connection.alias db_alias = schema_editor.connection.alias
app_registry.get_model("bookwyrm", "Connector").objects.using(db_alias).filter( app_registry.get_model("bookwyrm", "Connector").objects.using(db_alias).filter(
connector_file="self_connector" connector_file="self_connector"

View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.16 on 2022-11-15 21:40
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0163_merge_0160_auto_20221101_2251_0162_importjob_task_id"),
]
operations = [
migrations.AddField(
model_name="status",
name="ready",
field=models.BooleanField(default=True),
),
]

View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.16 on 2022-11-15 22:43
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0164_status_ready"),
]
operations = [
migrations.AlterField(
model_name="inviterequest",
name="answer",
field=models.TextField(blank=True, max_length=255, null=True),
),
]

View file

@ -0,0 +1,18 @@
# Generated by Django 3.2.16 on 2022-11-17 21:50
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0165_alter_inviterequest_answer"),
]
operations = [
migrations.AddField(
model_name="sitesettings",
name="imports_enabled",
field=models.BooleanField(default=True),
),
]

View file

@ -0,0 +1,23 @@
# Generated by Django 3.2.16 on 2022-11-25 19:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0166_sitesettings_imports_enabled"),
]
operations = [
migrations.AddField(
model_name="sitesettings",
name="impressum",
field=models.TextField(default="Add a impressum here."),
),
migrations.AddField(
model_name="sitesettings",
name="show_impressum",
field=models.BooleanField(default=False),
),
]

View file

@ -0,0 +1,23 @@
# Generated by Django 3.2.16 on 2022-12-05 13:53
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0166_sitesettings_imports_enabled"),
]
operations = [
migrations.AddField(
model_name="sitesettings",
name="import_size_limit",
field=models.IntegerField(default=0),
),
migrations.AddField(
model_name="sitesettings",
name="import_limit_reset",
field=models.IntegerField(default=0),
),
]

View file

@ -0,0 +1,28 @@
# Generated by Django 3.2.16 on 2022-12-05 17:01
import bookwyrm.models.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0167_auto_20221125_1900"),
]
operations = [
migrations.AddField(
model_name="author",
name="aasin",
field=bookwyrm.models.fields.CharField(
blank=True, max_length=255, null=True
),
),
migrations.AddField(
model_name="book",
name="aasin",
field=bookwyrm.models.fields.CharField(
blank=True, max_length=255, null=True
),
),
]

View file

@ -0,0 +1,63 @@
""" I added two new permission types and a new group to the management command that
creates the database on install, this creates them for existing instances """
# Generated by Django 3.2.16 on 2022-12-05 23:31
from django.db import migrations
def create_groups_and_perms(apps, schema_editor):
"""create the new "owner" group and "system admin" permission"""
db_alias = schema_editor.connection.alias
group_model = apps.get_model("auth", "Group")
# Add the "owner" group, if needed
owner_group, group_created = group_model.objects.using(db_alias).get_or_create(
name="owner"
)
# Create perms, if needed
user_model = apps.get_model("bookwyrm", "User")
content_type_model = apps.get_model("contenttypes", "ContentType")
content_type = content_type_model.objects.get_for_model(user_model)
perms_model = apps.get_model("auth", "Permission")
reg_perm, perm_created = perms_model.objects.using(db_alias).get_or_create(
codename="manage_registration",
name="allow or prevent user registration",
content_type=content_type,
)
admin_perm, admin_perm_created = perms_model.objects.using(db_alias).get_or_create(
codename="system_administration",
name="technical controls",
content_type=content_type,
)
# Add perms to the group if anything was created
if group_created or perm_created or admin_perm_created:
perms = [
"edit_instance_settings",
"set_user_group",
"control_federation",
"create_invites",
"moderate_user",
"moderate_post",
"edit_book",
]
owner_group.permissions.set(
perms_model.objects.using(db_alias).filter(codename__in=perms).all()
)
# also extend these perms to admins
# This is get or create so the tests don't fail -- it should already exist
admin_group, _ = group_model.objects.using(db_alias).get_or_create(name="admin")
admin_group.permissions.add(reg_perm)
admin_group.permissions.add(admin_perm)
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0167_auto_20221125_1900"),
]
operations = [
migrations.RunPython(create_groups_and_perms, migrations.RunPython.noop)
]

View file

@ -0,0 +1,28 @@
# Generated by Django 3.2.16 on 2022-12-06 09:02
import bookwyrm.models.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0168_auto_20221205_1701"),
]
operations = [
migrations.AddField(
model_name="author",
name="isfdb",
field=bookwyrm.models.fields.CharField(
blank=True, max_length=255, null=True
),
),
migrations.AddField(
model_name="book",
name="isfdb",
field=bookwyrm.models.fields.CharField(
blank=True, max_length=255, null=True
),
),
]

View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.16 on 2022-12-11 20:00
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0168_auto_20221205_2331"),
("bookwyrm", "0169_auto_20221206_0902"),
]
operations = []

View file

@ -0,0 +1,631 @@
# Generated by Django 3.2.16 on 2022-12-19 15:30
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
]
operations = [
migrations.AlterField(
model_name="user",
name="preferred_timezone",
field=models.CharField(
choices=[
("Africa/Abidjan", "Africa/Abidjan"),
("Africa/Accra", "Africa/Accra"),
("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
("Africa/Algiers", "Africa/Algiers"),
("Africa/Asmara", "Africa/Asmara"),
("Africa/Asmera", "Africa/Asmera"),
("Africa/Bamako", "Africa/Bamako"),
("Africa/Bangui", "Africa/Bangui"),
("Africa/Banjul", "Africa/Banjul"),
("Africa/Bissau", "Africa/Bissau"),
("Africa/Blantyre", "Africa/Blantyre"),
("Africa/Brazzaville", "Africa/Brazzaville"),
("Africa/Bujumbura", "Africa/Bujumbura"),
("Africa/Cairo", "Africa/Cairo"),
("Africa/Casablanca", "Africa/Casablanca"),
("Africa/Ceuta", "Africa/Ceuta"),
("Africa/Conakry", "Africa/Conakry"),
("Africa/Dakar", "Africa/Dakar"),
("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
("Africa/Djibouti", "Africa/Djibouti"),
("Africa/Douala", "Africa/Douala"),
("Africa/El_Aaiun", "Africa/El_Aaiun"),
("Africa/Freetown", "Africa/Freetown"),
("Africa/Gaborone", "Africa/Gaborone"),
("Africa/Harare", "Africa/Harare"),
("Africa/Johannesburg", "Africa/Johannesburg"),
("Africa/Juba", "Africa/Juba"),
("Africa/Kampala", "Africa/Kampala"),
("Africa/Khartoum", "Africa/Khartoum"),
("Africa/Kigali", "Africa/Kigali"),
("Africa/Kinshasa", "Africa/Kinshasa"),
("Africa/Lagos", "Africa/Lagos"),
("Africa/Libreville", "Africa/Libreville"),
("Africa/Lome", "Africa/Lome"),
("Africa/Luanda", "Africa/Luanda"),
("Africa/Lubumbashi", "Africa/Lubumbashi"),
("Africa/Lusaka", "Africa/Lusaka"),
("Africa/Malabo", "Africa/Malabo"),
("Africa/Maputo", "Africa/Maputo"),
("Africa/Maseru", "Africa/Maseru"),
("Africa/Mbabane", "Africa/Mbabane"),
("Africa/Mogadishu", "Africa/Mogadishu"),
("Africa/Monrovia", "Africa/Monrovia"),
("Africa/Nairobi", "Africa/Nairobi"),
("Africa/Ndjamena", "Africa/Ndjamena"),
("Africa/Niamey", "Africa/Niamey"),
("Africa/Nouakchott", "Africa/Nouakchott"),
("Africa/Ouagadougou", "Africa/Ouagadougou"),
("Africa/Porto-Novo", "Africa/Porto-Novo"),
("Africa/Sao_Tome", "Africa/Sao_Tome"),
("Africa/Timbuktu", "Africa/Timbuktu"),
("Africa/Tripoli", "Africa/Tripoli"),
("Africa/Tunis", "Africa/Tunis"),
("Africa/Windhoek", "Africa/Windhoek"),
("America/Adak", "America/Adak"),
("America/Anchorage", "America/Anchorage"),
("America/Anguilla", "America/Anguilla"),
("America/Antigua", "America/Antigua"),
("America/Araguaina", "America/Araguaina"),
(
"America/Argentina/Buenos_Aires",
"America/Argentina/Buenos_Aires",
),
("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
(
"America/Argentina/ComodRivadavia",
"America/Argentina/ComodRivadavia",
),
("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
(
"America/Argentina/Rio_Gallegos",
"America/Argentina/Rio_Gallegos",
),
("America/Argentina/Salta", "America/Argentina/Salta"),
("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
("America/Aruba", "America/Aruba"),
("America/Asuncion", "America/Asuncion"),
("America/Atikokan", "America/Atikokan"),
("America/Atka", "America/Atka"),
("America/Bahia", "America/Bahia"),
("America/Bahia_Banderas", "America/Bahia_Banderas"),
("America/Barbados", "America/Barbados"),
("America/Belem", "America/Belem"),
("America/Belize", "America/Belize"),
("America/Blanc-Sablon", "America/Blanc-Sablon"),
("America/Boa_Vista", "America/Boa_Vista"),
("America/Bogota", "America/Bogota"),
("America/Boise", "America/Boise"),
("America/Buenos_Aires", "America/Buenos_Aires"),
("America/Cambridge_Bay", "America/Cambridge_Bay"),
("America/Campo_Grande", "America/Campo_Grande"),
("America/Cancun", "America/Cancun"),
("America/Caracas", "America/Caracas"),
("America/Catamarca", "America/Catamarca"),
("America/Cayenne", "America/Cayenne"),
("America/Cayman", "America/Cayman"),
("America/Chicago", "America/Chicago"),
("America/Chihuahua", "America/Chihuahua"),
("America/Ciudad_Juarez", "America/Ciudad_Juarez"),
("America/Coral_Harbour", "America/Coral_Harbour"),
("America/Cordoba", "America/Cordoba"),
("America/Costa_Rica", "America/Costa_Rica"),
("America/Creston", "America/Creston"),
("America/Cuiaba", "America/Cuiaba"),
("America/Curacao", "America/Curacao"),
("America/Danmarkshavn", "America/Danmarkshavn"),
("America/Dawson", "America/Dawson"),
("America/Dawson_Creek", "America/Dawson_Creek"),
("America/Denver", "America/Denver"),
("America/Detroit", "America/Detroit"),
("America/Dominica", "America/Dominica"),
("America/Edmonton", "America/Edmonton"),
("America/Eirunepe", "America/Eirunepe"),
("America/El_Salvador", "America/El_Salvador"),
("America/Ensenada", "America/Ensenada"),
("America/Fort_Nelson", "America/Fort_Nelson"),
("America/Fort_Wayne", "America/Fort_Wayne"),
("America/Fortaleza", "America/Fortaleza"),
("America/Glace_Bay", "America/Glace_Bay"),
("America/Godthab", "America/Godthab"),
("America/Goose_Bay", "America/Goose_Bay"),
("America/Grand_Turk", "America/Grand_Turk"),
("America/Grenada", "America/Grenada"),
("America/Guadeloupe", "America/Guadeloupe"),
("America/Guatemala", "America/Guatemala"),
("America/Guayaquil", "America/Guayaquil"),
("America/Guyana", "America/Guyana"),
("America/Halifax", "America/Halifax"),
("America/Havana", "America/Havana"),
("America/Hermosillo", "America/Hermosillo"),
("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
("America/Indiana/Knox", "America/Indiana/Knox"),
("America/Indiana/Marengo", "America/Indiana/Marengo"),
("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
("America/Indiana/Vevay", "America/Indiana/Vevay"),
("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
("America/Indiana/Winamac", "America/Indiana/Winamac"),
("America/Indianapolis", "America/Indianapolis"),
("America/Inuvik", "America/Inuvik"),
("America/Iqaluit", "America/Iqaluit"),
("America/Jamaica", "America/Jamaica"),
("America/Jujuy", "America/Jujuy"),
("America/Juneau", "America/Juneau"),
("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
("America/Knox_IN", "America/Knox_IN"),
("America/Kralendijk", "America/Kralendijk"),
("America/La_Paz", "America/La_Paz"),
("America/Lima", "America/Lima"),
("America/Los_Angeles", "America/Los_Angeles"),
("America/Louisville", "America/Louisville"),
("America/Lower_Princes", "America/Lower_Princes"),
("America/Maceio", "America/Maceio"),
("America/Managua", "America/Managua"),
("America/Manaus", "America/Manaus"),
("America/Marigot", "America/Marigot"),
("America/Martinique", "America/Martinique"),
("America/Matamoros", "America/Matamoros"),
("America/Mazatlan", "America/Mazatlan"),
("America/Mendoza", "America/Mendoza"),
("America/Menominee", "America/Menominee"),
("America/Merida", "America/Merida"),
("America/Metlakatla", "America/Metlakatla"),
("America/Mexico_City", "America/Mexico_City"),
("America/Miquelon", "America/Miquelon"),
("America/Moncton", "America/Moncton"),
("America/Monterrey", "America/Monterrey"),
("America/Montevideo", "America/Montevideo"),
("America/Montreal", "America/Montreal"),
("America/Montserrat", "America/Montserrat"),
("America/Nassau", "America/Nassau"),
("America/New_York", "America/New_York"),
("America/Nipigon", "America/Nipigon"),
("America/Nome", "America/Nome"),
("America/Noronha", "America/Noronha"),
("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
("America/North_Dakota/Center", "America/North_Dakota/Center"),
(
"America/North_Dakota/New_Salem",
"America/North_Dakota/New_Salem",
),
("America/Nuuk", "America/Nuuk"),
("America/Ojinaga", "America/Ojinaga"),
("America/Panama", "America/Panama"),
("America/Pangnirtung", "America/Pangnirtung"),
("America/Paramaribo", "America/Paramaribo"),
("America/Phoenix", "America/Phoenix"),
("America/Port-au-Prince", "America/Port-au-Prince"),
("America/Port_of_Spain", "America/Port_of_Spain"),
("America/Porto_Acre", "America/Porto_Acre"),
("America/Porto_Velho", "America/Porto_Velho"),
("America/Puerto_Rico", "America/Puerto_Rico"),
("America/Punta_Arenas", "America/Punta_Arenas"),
("America/Rainy_River", "America/Rainy_River"),
("America/Rankin_Inlet", "America/Rankin_Inlet"),
("America/Recife", "America/Recife"),
("America/Regina", "America/Regina"),
("America/Resolute", "America/Resolute"),
("America/Rio_Branco", "America/Rio_Branco"),
("America/Rosario", "America/Rosario"),
("America/Santa_Isabel", "America/Santa_Isabel"),
("America/Santarem", "America/Santarem"),
("America/Santiago", "America/Santiago"),
("America/Santo_Domingo", "America/Santo_Domingo"),
("America/Sao_Paulo", "America/Sao_Paulo"),
("America/Scoresbysund", "America/Scoresbysund"),
("America/Shiprock", "America/Shiprock"),
("America/Sitka", "America/Sitka"),
("America/St_Barthelemy", "America/St_Barthelemy"),
("America/St_Johns", "America/St_Johns"),
("America/St_Kitts", "America/St_Kitts"),
("America/St_Lucia", "America/St_Lucia"),
("America/St_Thomas", "America/St_Thomas"),
("America/St_Vincent", "America/St_Vincent"),
("America/Swift_Current", "America/Swift_Current"),
("America/Tegucigalpa", "America/Tegucigalpa"),
("America/Thule", "America/Thule"),
("America/Thunder_Bay", "America/Thunder_Bay"),
("America/Tijuana", "America/Tijuana"),
("America/Toronto", "America/Toronto"),
("America/Tortola", "America/Tortola"),
("America/Vancouver", "America/Vancouver"),
("America/Virgin", "America/Virgin"),
("America/Whitehorse", "America/Whitehorse"),
("America/Winnipeg", "America/Winnipeg"),
("America/Yakutat", "America/Yakutat"),
("America/Yellowknife", "America/Yellowknife"),
("Antarctica/Casey", "Antarctica/Casey"),
("Antarctica/Davis", "Antarctica/Davis"),
("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
("Antarctica/Macquarie", "Antarctica/Macquarie"),
("Antarctica/Mawson", "Antarctica/Mawson"),
("Antarctica/McMurdo", "Antarctica/McMurdo"),
("Antarctica/Palmer", "Antarctica/Palmer"),
("Antarctica/Rothera", "Antarctica/Rothera"),
("Antarctica/South_Pole", "Antarctica/South_Pole"),
("Antarctica/Syowa", "Antarctica/Syowa"),
("Antarctica/Troll", "Antarctica/Troll"),
("Antarctica/Vostok", "Antarctica/Vostok"),
("Arctic/Longyearbyen", "Arctic/Longyearbyen"),
("Asia/Aden", "Asia/Aden"),
("Asia/Almaty", "Asia/Almaty"),
("Asia/Amman", "Asia/Amman"),
("Asia/Anadyr", "Asia/Anadyr"),
("Asia/Aqtau", "Asia/Aqtau"),
("Asia/Aqtobe", "Asia/Aqtobe"),
("Asia/Ashgabat", "Asia/Ashgabat"),
("Asia/Ashkhabad", "Asia/Ashkhabad"),
("Asia/Atyrau", "Asia/Atyrau"),
("Asia/Baghdad", "Asia/Baghdad"),
("Asia/Bahrain", "Asia/Bahrain"),
("Asia/Baku", "Asia/Baku"),
("Asia/Bangkok", "Asia/Bangkok"),
("Asia/Barnaul", "Asia/Barnaul"),
("Asia/Beirut", "Asia/Beirut"),
("Asia/Bishkek", "Asia/Bishkek"),
("Asia/Brunei", "Asia/Brunei"),
("Asia/Calcutta", "Asia/Calcutta"),
("Asia/Chita", "Asia/Chita"),
("Asia/Choibalsan", "Asia/Choibalsan"),
("Asia/Chongqing", "Asia/Chongqing"),
("Asia/Chungking", "Asia/Chungking"),
("Asia/Colombo", "Asia/Colombo"),
("Asia/Dacca", "Asia/Dacca"),
("Asia/Damascus", "Asia/Damascus"),
("Asia/Dhaka", "Asia/Dhaka"),
("Asia/Dili", "Asia/Dili"),
("Asia/Dubai", "Asia/Dubai"),
("Asia/Dushanbe", "Asia/Dushanbe"),
("Asia/Famagusta", "Asia/Famagusta"),
("Asia/Gaza", "Asia/Gaza"),
("Asia/Harbin", "Asia/Harbin"),
("Asia/Hebron", "Asia/Hebron"),
("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
("Asia/Hong_Kong", "Asia/Hong_Kong"),
("Asia/Hovd", "Asia/Hovd"),
("Asia/Irkutsk", "Asia/Irkutsk"),
("Asia/Istanbul", "Asia/Istanbul"),
("Asia/Jakarta", "Asia/Jakarta"),
("Asia/Jayapura", "Asia/Jayapura"),
("Asia/Jerusalem", "Asia/Jerusalem"),
("Asia/Kabul", "Asia/Kabul"),
("Asia/Kamchatka", "Asia/Kamchatka"),
("Asia/Karachi", "Asia/Karachi"),
("Asia/Kashgar", "Asia/Kashgar"),
("Asia/Kathmandu", "Asia/Kathmandu"),
("Asia/Katmandu", "Asia/Katmandu"),
("Asia/Khandyga", "Asia/Khandyga"),
("Asia/Kolkata", "Asia/Kolkata"),
("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
("Asia/Kuching", "Asia/Kuching"),
("Asia/Kuwait", "Asia/Kuwait"),
("Asia/Macao", "Asia/Macao"),
("Asia/Macau", "Asia/Macau"),
("Asia/Magadan", "Asia/Magadan"),
("Asia/Makassar", "Asia/Makassar"),
("Asia/Manila", "Asia/Manila"),
("Asia/Muscat", "Asia/Muscat"),
("Asia/Nicosia", "Asia/Nicosia"),
("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
("Asia/Novosibirsk", "Asia/Novosibirsk"),
("Asia/Omsk", "Asia/Omsk"),
("Asia/Oral", "Asia/Oral"),
("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
("Asia/Pontianak", "Asia/Pontianak"),
("Asia/Pyongyang", "Asia/Pyongyang"),
("Asia/Qatar", "Asia/Qatar"),
("Asia/Qostanay", "Asia/Qostanay"),
("Asia/Qyzylorda", "Asia/Qyzylorda"),
("Asia/Rangoon", "Asia/Rangoon"),
("Asia/Riyadh", "Asia/Riyadh"),
("Asia/Saigon", "Asia/Saigon"),
("Asia/Sakhalin", "Asia/Sakhalin"),
("Asia/Samarkand", "Asia/Samarkand"),
("Asia/Seoul", "Asia/Seoul"),
("Asia/Shanghai", "Asia/Shanghai"),
("Asia/Singapore", "Asia/Singapore"),
("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
("Asia/Taipei", "Asia/Taipei"),
("Asia/Tashkent", "Asia/Tashkent"),
("Asia/Tbilisi", "Asia/Tbilisi"),
("Asia/Tehran", "Asia/Tehran"),
("Asia/Tel_Aviv", "Asia/Tel_Aviv"),
("Asia/Thimbu", "Asia/Thimbu"),
("Asia/Thimphu", "Asia/Thimphu"),
("Asia/Tokyo", "Asia/Tokyo"),
("Asia/Tomsk", "Asia/Tomsk"),
("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"),
("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
("Asia/Ulan_Bator", "Asia/Ulan_Bator"),
("Asia/Urumqi", "Asia/Urumqi"),
("Asia/Ust-Nera", "Asia/Ust-Nera"),
("Asia/Vientiane", "Asia/Vientiane"),
("Asia/Vladivostok", "Asia/Vladivostok"),
("Asia/Yakutsk", "Asia/Yakutsk"),
("Asia/Yangon", "Asia/Yangon"),
("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
("Asia/Yerevan", "Asia/Yerevan"),
("Atlantic/Azores", "Atlantic/Azores"),
("Atlantic/Bermuda", "Atlantic/Bermuda"),
("Atlantic/Canary", "Atlantic/Canary"),
("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
("Atlantic/Faeroe", "Atlantic/Faeroe"),
("Atlantic/Faroe", "Atlantic/Faroe"),
("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"),
("Atlantic/Madeira", "Atlantic/Madeira"),
("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
("Atlantic/St_Helena", "Atlantic/St_Helena"),
("Atlantic/Stanley", "Atlantic/Stanley"),
("Australia/ACT", "Australia/ACT"),
("Australia/Adelaide", "Australia/Adelaide"),
("Australia/Brisbane", "Australia/Brisbane"),
("Australia/Broken_Hill", "Australia/Broken_Hill"),
("Australia/Canberra", "Australia/Canberra"),
("Australia/Currie", "Australia/Currie"),
("Australia/Darwin", "Australia/Darwin"),
("Australia/Eucla", "Australia/Eucla"),
("Australia/Hobart", "Australia/Hobart"),
("Australia/LHI", "Australia/LHI"),
("Australia/Lindeman", "Australia/Lindeman"),
("Australia/Lord_Howe", "Australia/Lord_Howe"),
("Australia/Melbourne", "Australia/Melbourne"),
("Australia/NSW", "Australia/NSW"),
("Australia/North", "Australia/North"),
("Australia/Perth", "Australia/Perth"),
("Australia/Queensland", "Australia/Queensland"),
("Australia/South", "Australia/South"),
("Australia/Sydney", "Australia/Sydney"),
("Australia/Tasmania", "Australia/Tasmania"),
("Australia/Victoria", "Australia/Victoria"),
("Australia/West", "Australia/West"),
("Australia/Yancowinna", "Australia/Yancowinna"),
("Brazil/Acre", "Brazil/Acre"),
("Brazil/DeNoronha", "Brazil/DeNoronha"),
("Brazil/East", "Brazil/East"),
("Brazil/West", "Brazil/West"),
("CET", "CET"),
("CST6CDT", "CST6CDT"),
("Canada/Atlantic", "Canada/Atlantic"),
("Canada/Central", "Canada/Central"),
("Canada/Eastern", "Canada/Eastern"),
("Canada/Mountain", "Canada/Mountain"),
("Canada/Newfoundland", "Canada/Newfoundland"),
("Canada/Pacific", "Canada/Pacific"),
("Canada/Saskatchewan", "Canada/Saskatchewan"),
("Canada/Yukon", "Canada/Yukon"),
("Chile/Continental", "Chile/Continental"),
("Chile/EasterIsland", "Chile/EasterIsland"),
("Cuba", "Cuba"),
("EET", "EET"),
("EST", "EST"),
("EST5EDT", "EST5EDT"),
("Egypt", "Egypt"),
("Eire", "Eire"),
("Etc/GMT", "Etc/GMT"),
("Etc/GMT+0", "Etc/GMT+0"),
("Etc/GMT+1", "Etc/GMT+1"),
("Etc/GMT+10", "Etc/GMT+10"),
("Etc/GMT+11", "Etc/GMT+11"),
("Etc/GMT+12", "Etc/GMT+12"),
("Etc/GMT+2", "Etc/GMT+2"),
("Etc/GMT+3", "Etc/GMT+3"),
("Etc/GMT+4", "Etc/GMT+4"),
("Etc/GMT+5", "Etc/GMT+5"),
("Etc/GMT+6", "Etc/GMT+6"),
("Etc/GMT+7", "Etc/GMT+7"),
("Etc/GMT+8", "Etc/GMT+8"),
("Etc/GMT+9", "Etc/GMT+9"),
("Etc/GMT-0", "Etc/GMT-0"),
("Etc/GMT-1", "Etc/GMT-1"),
("Etc/GMT-10", "Etc/GMT-10"),
("Etc/GMT-11", "Etc/GMT-11"),
("Etc/GMT-12", "Etc/GMT-12"),
("Etc/GMT-13", "Etc/GMT-13"),
("Etc/GMT-14", "Etc/GMT-14"),
("Etc/GMT-2", "Etc/GMT-2"),
("Etc/GMT-3", "Etc/GMT-3"),
("Etc/GMT-4", "Etc/GMT-4"),
("Etc/GMT-5", "Etc/GMT-5"),
("Etc/GMT-6", "Etc/GMT-6"),
("Etc/GMT-7", "Etc/GMT-7"),
("Etc/GMT-8", "Etc/GMT-8"),
("Etc/GMT-9", "Etc/GMT-9"),
("Etc/GMT0", "Etc/GMT0"),
("Etc/Greenwich", "Etc/Greenwich"),
("Etc/UCT", "Etc/UCT"),
("Etc/UTC", "Etc/UTC"),
("Etc/Universal", "Etc/Universal"),
("Etc/Zulu", "Etc/Zulu"),
("Europe/Amsterdam", "Europe/Amsterdam"),
("Europe/Andorra", "Europe/Andorra"),
("Europe/Astrakhan", "Europe/Astrakhan"),
("Europe/Athens", "Europe/Athens"),
("Europe/Belfast", "Europe/Belfast"),
("Europe/Belgrade", "Europe/Belgrade"),
("Europe/Berlin", "Europe/Berlin"),
("Europe/Bratislava", "Europe/Bratislava"),
("Europe/Brussels", "Europe/Brussels"),
("Europe/Bucharest", "Europe/Bucharest"),
("Europe/Budapest", "Europe/Budapest"),
("Europe/Busingen", "Europe/Busingen"),
("Europe/Chisinau", "Europe/Chisinau"),
("Europe/Copenhagen", "Europe/Copenhagen"),
("Europe/Dublin", "Europe/Dublin"),
("Europe/Gibraltar", "Europe/Gibraltar"),
("Europe/Guernsey", "Europe/Guernsey"),
("Europe/Helsinki", "Europe/Helsinki"),
("Europe/Isle_of_Man", "Europe/Isle_of_Man"),
("Europe/Istanbul", "Europe/Istanbul"),
("Europe/Jersey", "Europe/Jersey"),
("Europe/Kaliningrad", "Europe/Kaliningrad"),
("Europe/Kiev", "Europe/Kiev"),
("Europe/Kirov", "Europe/Kirov"),
("Europe/Kyiv", "Europe/Kyiv"),
("Europe/Lisbon", "Europe/Lisbon"),
("Europe/Ljubljana", "Europe/Ljubljana"),
("Europe/London", "Europe/London"),
("Europe/Luxembourg", "Europe/Luxembourg"),
("Europe/Madrid", "Europe/Madrid"),
("Europe/Malta", "Europe/Malta"),
("Europe/Mariehamn", "Europe/Mariehamn"),
("Europe/Minsk", "Europe/Minsk"),
("Europe/Monaco", "Europe/Monaco"),
("Europe/Moscow", "Europe/Moscow"),
("Europe/Nicosia", "Europe/Nicosia"),
("Europe/Oslo", "Europe/Oslo"),
("Europe/Paris", "Europe/Paris"),
("Europe/Podgorica", "Europe/Podgorica"),
("Europe/Prague", "Europe/Prague"),
("Europe/Riga", "Europe/Riga"),
("Europe/Rome", "Europe/Rome"),
("Europe/Samara", "Europe/Samara"),
("Europe/San_Marino", "Europe/San_Marino"),
("Europe/Sarajevo", "Europe/Sarajevo"),
("Europe/Saratov", "Europe/Saratov"),
("Europe/Simferopol", "Europe/Simferopol"),
("Europe/Skopje", "Europe/Skopje"),
("Europe/Sofia", "Europe/Sofia"),
("Europe/Stockholm", "Europe/Stockholm"),
("Europe/Tallinn", "Europe/Tallinn"),
("Europe/Tirane", "Europe/Tirane"),
("Europe/Tiraspol", "Europe/Tiraspol"),
("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
("Europe/Uzhgorod", "Europe/Uzhgorod"),
("Europe/Vaduz", "Europe/Vaduz"),
("Europe/Vatican", "Europe/Vatican"),
("Europe/Vienna", "Europe/Vienna"),
("Europe/Vilnius", "Europe/Vilnius"),
("Europe/Volgograd", "Europe/Volgograd"),
("Europe/Warsaw", "Europe/Warsaw"),
("Europe/Zagreb", "Europe/Zagreb"),
("Europe/Zaporozhye", "Europe/Zaporozhye"),
("Europe/Zurich", "Europe/Zurich"),
("GB", "GB"),
("GB-Eire", "GB-Eire"),
("GMT", "GMT"),
("GMT+0", "GMT+0"),
("GMT-0", "GMT-0"),
("GMT0", "GMT0"),
("Greenwich", "Greenwich"),
("HST", "HST"),
("Hongkong", "Hongkong"),
("Iceland", "Iceland"),
("Indian/Antananarivo", "Indian/Antananarivo"),
("Indian/Chagos", "Indian/Chagos"),
("Indian/Christmas", "Indian/Christmas"),
("Indian/Cocos", "Indian/Cocos"),
("Indian/Comoro", "Indian/Comoro"),
("Indian/Kerguelen", "Indian/Kerguelen"),
("Indian/Mahe", "Indian/Mahe"),
("Indian/Maldives", "Indian/Maldives"),
("Indian/Mauritius", "Indian/Mauritius"),
("Indian/Mayotte", "Indian/Mayotte"),
("Indian/Reunion", "Indian/Reunion"),
("Iran", "Iran"),
("Israel", "Israel"),
("Jamaica", "Jamaica"),
("Japan", "Japan"),
("Kwajalein", "Kwajalein"),
("Libya", "Libya"),
("MET", "MET"),
("MST", "MST"),
("MST7MDT", "MST7MDT"),
("Mexico/BajaNorte", "Mexico/BajaNorte"),
("Mexico/BajaSur", "Mexico/BajaSur"),
("Mexico/General", "Mexico/General"),
("NZ", "NZ"),
("NZ-CHAT", "NZ-CHAT"),
("Navajo", "Navajo"),
("PRC", "PRC"),
("PST8PDT", "PST8PDT"),
("Pacific/Apia", "Pacific/Apia"),
("Pacific/Auckland", "Pacific/Auckland"),
("Pacific/Bougainville", "Pacific/Bougainville"),
("Pacific/Chatham", "Pacific/Chatham"),
("Pacific/Chuuk", "Pacific/Chuuk"),
("Pacific/Easter", "Pacific/Easter"),
("Pacific/Efate", "Pacific/Efate"),
("Pacific/Enderbury", "Pacific/Enderbury"),
("Pacific/Fakaofo", "Pacific/Fakaofo"),
("Pacific/Fiji", "Pacific/Fiji"),
("Pacific/Funafuti", "Pacific/Funafuti"),
("Pacific/Galapagos", "Pacific/Galapagos"),
("Pacific/Gambier", "Pacific/Gambier"),
("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
("Pacific/Guam", "Pacific/Guam"),
("Pacific/Honolulu", "Pacific/Honolulu"),
("Pacific/Johnston", "Pacific/Johnston"),
("Pacific/Kanton", "Pacific/Kanton"),
("Pacific/Kiritimati", "Pacific/Kiritimati"),
("Pacific/Kosrae", "Pacific/Kosrae"),
("Pacific/Kwajalein", "Pacific/Kwajalein"),
("Pacific/Majuro", "Pacific/Majuro"),
("Pacific/Marquesas", "Pacific/Marquesas"),
("Pacific/Midway", "Pacific/Midway"),
("Pacific/Nauru", "Pacific/Nauru"),
("Pacific/Niue", "Pacific/Niue"),
("Pacific/Norfolk", "Pacific/Norfolk"),
("Pacific/Noumea", "Pacific/Noumea"),
("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
("Pacific/Palau", "Pacific/Palau"),
("Pacific/Pitcairn", "Pacific/Pitcairn"),
("Pacific/Pohnpei", "Pacific/Pohnpei"),
("Pacific/Ponape", "Pacific/Ponape"),
("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
("Pacific/Rarotonga", "Pacific/Rarotonga"),
("Pacific/Saipan", "Pacific/Saipan"),
("Pacific/Samoa", "Pacific/Samoa"),
("Pacific/Tahiti", "Pacific/Tahiti"),
("Pacific/Tarawa", "Pacific/Tarawa"),
("Pacific/Tongatapu", "Pacific/Tongatapu"),
("Pacific/Truk", "Pacific/Truk"),
("Pacific/Wake", "Pacific/Wake"),
("Pacific/Wallis", "Pacific/Wallis"),
("Pacific/Yap", "Pacific/Yap"),
("Poland", "Poland"),
("Portugal", "Portugal"),
("ROC", "ROC"),
("ROK", "ROK"),
("Singapore", "Singapore"),
("Turkey", "Turkey"),
("UCT", "UCT"),
("US/Alaska", "US/Alaska"),
("US/Aleutian", "US/Aleutian"),
("US/Arizona", "US/Arizona"),
("US/Central", "US/Central"),
("US/East-Indiana", "US/East-Indiana"),
("US/Eastern", "US/Eastern"),
("US/Hawaii", "US/Hawaii"),
("US/Indiana-Starke", "US/Indiana-Starke"),
("US/Michigan", "US/Michigan"),
("US/Mountain", "US/Mountain"),
("US/Pacific", "US/Pacific"),
("US/Samoa", "US/Samoa"),
("UTC", "UTC"),
("Universal", "Universal"),
("W-SU", "W-SU"),
("WET", "WET"),
("Zulu", "Zulu"),
],
default="UTC",
max_length=255,
),
),
]

View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.16 on 2022-12-19 20:20
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0167_sitesettings_import_size_limit"),
("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
]
operations = []

View file

@ -0,0 +1,42 @@
# Generated by Django 3.2.16 on 2022-12-21 18:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0171_alter_user_preferred_timezone"),
]
operations = [
migrations.AlterField(
model_name="user",
name="preferred_language",
field=models.CharField(
blank=True,
choices=[
("en-us", "English"),
("ca-es", "Català (Catalan)"),
("de-de", "Deutsch (German)"),
("es-es", "Español (Spanish)"),
("eu-es", "Euskara (Basque)"),
("gl-es", "Galego (Galician)"),
("it-it", "Italiano (Italian)"),
("fi-fi", "Suomi (Finnish)"),
("fr-fr", "Français (French)"),
("lt-lt", "Lietuvių (Lithuanian)"),
("no-no", "Norsk (Norwegian)"),
("pl-pl", "Polski (Polish)"),
("pt-br", "Português do Brasil (Brazilian Portuguese)"),
("pt-pt", "Português Europeu (European Portuguese)"),
("ro-ro", "Română (Romanian)"),
("sv-se", "Svenska (Swedish)"),
("zh-hans", "简体中文 (Simplified Chinese)"),
("zh-hant", "繁體中文 (Traditional Chinese)"),
],
max_length=255,
null=True,
),
),
]

View file

@ -0,0 +1,21 @@
# Generated by Django 3.2.16 on 2023-01-15 08:38
import bookwyrm.models.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0172_alter_user_preferred_language"),
]
operations = [
migrations.AddField(
model_name="author",
name="website",
field=bookwyrm.models.fields.CharField(
blank=True, max_length=255, null=True
),
),
]

View file

@ -0,0 +1,34 @@
# Generated by Django 3.2.16 on 2022-12-27 21:34
from django.db import migrations, models
import django.db.models.deletion
def backfill_sitesettings(apps, schema_editor):
db_alias = schema_editor.connection.alias
group_model = apps.get_model("auth", "Group")
editor_group = group_model.objects.using(db_alias).filter(name="editor").first()
sitesettings_model = apps.get_model("bookwyrm", "SiteSettings")
sitesettings_model.objects.update(default_user_auth_group=editor_group)
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0175_merge_0173_author_website_0174_merge_20230111_1523"),
]
operations = [
migrations.AddField(
model_name="sitesettings",
name="default_user_auth_group",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.RESTRICT,
to="auth.group",
),
),
migrations.RunPython(backfill_sitesettings, migrations.RunPython.noop),
]

View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.16 on 2023-01-02 14:44
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0171_merge_20221219_2020"),
("bookwyrm", "0172_alter_user_preferred_language"),
]
operations = []

View file

@ -0,0 +1,35 @@
# Generated by Django 3.2.16 on 2023-01-30 12:40
import django.core.validators
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("auth", "0012_alter_user_first_name_max_length"),
("bookwyrm", "0173_default_user_auth_group_setting"),
]
operations = [
migrations.AddField(
model_name="quotation",
name="endposition",
field=models.IntegerField(
blank=True,
null=True,
validators=[django.core.validators.MinValueValidator(0)],
),
),
migrations.AlterField(
model_name="sitesettings",
name="default_user_auth_group",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="auth.group",
),
),
]

View file

@ -0,0 +1,46 @@
# Generated by Django 3.2.18 on 2023-02-22 17:42
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0174_auto_20230130_1240"),
]
operations = [
migrations.AddField(
model_name="notification",
name="related_link_domains",
field=models.ManyToManyField(to="bookwyrm.LinkDomain"),
),
migrations.AlterField(
model_name="notification",
name="notification_type",
field=models.CharField(
choices=[
("FAVORITE", "Favorite"),
("REPLY", "Reply"),
("MENTION", "Mention"),
("TAG", "Tag"),
("FOLLOW", "Follow"),
("FOLLOW_REQUEST", "Follow Request"),
("BOOST", "Boost"),
("IMPORT", "Import"),
("ADD", "Add"),
("REPORT", "Report"),
("LINK_DOMAIN", "Link Domain"),
("INVITE", "Invite"),
("ACCEPT", "Accept"),
("JOIN", "Join"),
("LEAVE", "Leave"),
("REMOVE", "Remove"),
("GROUP_PRIVACY", "Group Privacy"),
("GROUP_NAME", "Group Name"),
("GROUP_DESCRIPTION", "Group Description"),
],
max_length=255,
),
),
]

View file

@ -0,0 +1,12 @@
# Generated by Django 3.2.16 on 2023-01-11 15:23
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0173_merge_20230102_1444"),
]
operations = []

View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.16 on 2023-01-19 20:17
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0173_author_website"),
("bookwyrm", "0174_merge_20230111_1523"),
]
operations = []

View file

@ -0,0 +1,53 @@
# Generated by Django 3.2.16 on 2022-12-17 19:28
import bookwyrm.models.fields
import django.contrib.postgres.fields.citext
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0174_auto_20230130_1240"),
]
operations = [
migrations.CreateModel(
name="Hashtag",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created_date", models.DateTimeField(auto_now_add=True)),
("updated_date", models.DateTimeField(auto_now=True)),
(
"remote_id",
bookwyrm.models.fields.RemoteIdField(
max_length=255,
null=True,
validators=[bookwyrm.models.fields.validate_remote_id],
),
),
(
"name",
django.contrib.postgres.fields.citext.CICharField(max_length=256),
),
],
options={
"abstract": False,
},
),
migrations.AddField(
model_name="status",
name="mention_hashtags",
field=bookwyrm.models.fields.TagField(
related_name="mention_hashtag", to="bookwyrm.Hashtag"
),
),
]

View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.18 on 2023-03-12 23:41
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0174_auto_20230222_1742"),
("bookwyrm", "0176_hashtag_support"),
]
operations = []

View file

@ -0,0 +1,61 @@
# Generated by Django 3.2.18 on 2023-03-28 21:32
import bookwyrm.models.fields
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("auth", "0012_alter_user_first_name_max_length"),
("bookwyrm", "0177_merge_0174_auto_20230222_1742_0176_hashtag_support"),
]
operations = [
migrations.AlterField(
model_name="hashtag",
name="name",
field=bookwyrm.models.fields.CICharField(max_length=256),
),
migrations.AlterField(
model_name="sitesettings",
name="default_user_auth_group",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.RESTRICT,
to="auth.group",
),
),
migrations.AlterField(
model_name="user",
name="preferred_language",
field=models.CharField(
blank=True,
choices=[
("en-us", "English"),
("ca-es", "Català (Catalan)"),
("de-de", "Deutsch (German)"),
("eo-uy", "Esperanto (Esperanto)"),
("es-es", "Español (Spanish)"),
("eu-es", "Euskara (Basque)"),
("gl-es", "Galego (Galician)"),
("it-it", "Italiano (Italian)"),
("fi-fi", "Suomi (Finnish)"),
("fr-fr", "Français (French)"),
("lt-lt", "Lietuvių (Lithuanian)"),
("no-no", "Norsk (Norwegian)"),
("pl-pl", "Polski (Polish)"),
("pt-br", "Português do Brasil (Brazilian Portuguese)"),
("pt-pt", "Português Europeu (European Portuguese)"),
("ro-ro", "Română (Romanian)"),
("sv-se", "Svenska (Swedish)"),
("zh-hans", "简体中文 (Simplified Chinese)"),
("zh-hant", "繁體中文 (Traditional Chinese)"),
],
max_length=255,
null=True,
),
),
]

View file

@ -0,0 +1,49 @@
import re
from itertools import chain
from django.db import migrations, transaction
from django.db.models import Q
from bookwyrm.settings import LANGUAGE_ARTICLES
def set_sort_title(edition):
articles = chain(
*(LANGUAGE_ARTICLES.get(language, ()) for language in tuple(edition.languages))
)
edition.sort_title = re.sub(
f'^{" |^".join(articles)} ', "", str(edition.title).lower()
)
return edition
@transaction.atomic
def populate_sort_title(apps, schema_editor):
Edition = apps.get_model("bookwyrm", "Edition")
db_alias = schema_editor.connection.alias
editions_wo_sort_title = Edition.objects.using(db_alias).filter(
Q(sort_title__isnull=True) | Q(sort_title__exact="")
)
batch_size = 1000
start = 0
end = batch_size
while True:
batch = editions_wo_sort_title[start:end]
if not batch.exists():
break
Edition.objects.bulk_update(
(set_sort_title(edition) for edition in batch), ["sort_title"]
)
start = end
end += batch_size
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0178_auto_20230328_2132"),
]
operations = [
migrations.RunPython(populate_sort_title),
]

View file

@ -0,0 +1,36 @@
# Generated by Django 3.2.18 on 2023-05-16 16:02
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0178_auto_20230328_2132"),
]
operations = [
migrations.AddField(
model_name="reportcomment",
name="action_type",
field=models.CharField(
choices=[
("comment", "Comment"),
("resolve", "Resolved report"),
("reopen", "Re-opened report"),
("message_reporter", "Messaged reporter"),
("message_offender", "Messaged reported user"),
("user_suspension", "Suspended user"),
("user_unsuspension", "Un-suspended user"),
("user_perms", "Changed user permission level"),
("user_deletion", "Deleted user account"),
("block_domain", "Blocked domain"),
("approve_domain", "Approved domain"),
("delete_item", "Deleted item"),
],
default="comment",
max_length=20,
),
),
migrations.RenameModel("ReportComment", "ReportAction"),
]

View file

@ -0,0 +1,17 @@
# Generated by Django 3.2.18 on 2023-06-21 22:01
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0179_reportcomment_comment_type"),
]
operations = [
migrations.AlterModelOptions(
name="reportaction",
options={"ordering": ("created_date",)},
),
]

View file

@ -0,0 +1,44 @@
# Generated by Django 3.2.19 on 2023-07-23 19:33
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0179_populate_sort_title"),
]
operations = [
migrations.AlterField(
model_name="user",
name="preferred_language",
field=models.CharField(
blank=True,
choices=[
("en-us", "English"),
("ca-es", "Català (Catalan)"),
("de-de", "Deutsch (German)"),
("eo-uy", "Esperanto (Esperanto)"),
("es-es", "Español (Spanish)"),
("eu-es", "Euskara (Basque)"),
("gl-es", "Galego (Galician)"),
("it-it", "Italiano (Italian)"),
("fi-fi", "Suomi (Finnish)"),
("fr-fr", "Français (French)"),
("lt-lt", "Lietuvių (Lithuanian)"),
("nl-nl", "Nederlands (Dutch)"),
("no-no", "Norsk (Norwegian)"),
("pl-pl", "Polski (Polish)"),
("pt-br", "Português do Brasil (Brazilian Portuguese)"),
("pt-pt", "Português Europeu (European Portuguese)"),
("ro-ro", "Română (Romanian)"),
("sv-se", "Svenska (Swedish)"),
("zh-hans", "简体中文 (Simplified Chinese)"),
("zh-hant", "繁體中文 (Traditional Chinese)"),
],
max_length=255,
null=True,
),
),
]

View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.20 on 2023-08-06 23:02
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0180_alter_reportaction_options"),
("bookwyrm", "0180_alter_user_preferred_language"),
]
operations = []

View file

@ -20,7 +20,7 @@ from .readthrough import ReadThrough, ProgressUpdate, ProgressMode
from .user import User, KeyPair from .user import User, KeyPair
from .annual_goal import AnnualGoal from .annual_goal import AnnualGoal
from .relationship import UserFollows, UserFollowRequest, UserBlocks from .relationship import UserFollows, UserFollowRequest, UserBlocks
from .report import Report, ReportComment from .report import Report, ReportAction
from .federated_server import FederatedServer from .federated_server import FederatedServer
from .group import Group, GroupMember, GroupMemberInvitation from .group import Group, GroupMember, GroupMemberInvitation
@ -34,6 +34,8 @@ from .antispam import EmailBlocklist, IPBlocklist, AutoMod, automod_task
from .notification import Notification from .notification import Notification
from .hashtag import Hashtag
cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass) cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
activity_models = { activity_models = {
c[1].activity_serializer.__name__: c[1] c[1].activity_serializer.__name__: c[1]

View file

@ -6,8 +6,9 @@ from functools import reduce
import json import json
import operator import operator
import logging import logging
from typing import List from typing import Any, Optional
from uuid import uuid4 from uuid import uuid4
from typing_extensions import Self
import aiohttp import aiohttp
from Crypto.PublicKey import RSA from Crypto.PublicKey import RSA
@ -21,11 +22,11 @@ from django.utils.http import http_date
from bookwyrm import activitypub from bookwyrm import activitypub
from bookwyrm.settings import USER_AGENT, PAGE_LENGTH from bookwyrm.settings import USER_AGENT, PAGE_LENGTH
from bookwyrm.signatures import make_signature, make_digest from bookwyrm.signatures import make_signature, make_digest
from bookwyrm.tasks import app, MEDIUM from bookwyrm.tasks import app, BROADCAST
from bookwyrm.models.fields import ImageField, ManyToManyField from bookwyrm.models.fields import ImageField, ManyToManyField
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# I tried to separate these classes into mutliple files but I kept getting # I tried to separate these classes into multiple files but I kept getting
# circular import errors so I gave up. I'm sure it could be done though! # circular import errors so I gave up. I'm sure it could be done though!
PropertyField = namedtuple("PropertyField", ("set_activity_from_field")) PropertyField = namedtuple("PropertyField", ("set_activity_from_field"))
@ -85,13 +86,13 @@ class ActivitypubMixin:
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@classmethod @classmethod
def find_existing_by_remote_id(cls, remote_id): def find_existing_by_remote_id(cls, remote_id: str) -> Self:
"""look up a remote id in the db""" """look up a remote id in the db"""
return cls.find_existing({"id": remote_id}) return cls.find_existing({"id": remote_id})
@classmethod @classmethod
def find_existing(cls, data): def find_existing(cls, data):
"""compare data to fields that can be used for deduplation. """compare data to fields that can be used for deduplication.
This always includes remote_id, but can also be unique identifiers This always includes remote_id, but can also be unique identifiers
like an isbn for an edition""" like an isbn for an edition"""
filters = [] filters = []
@ -126,7 +127,7 @@ class ActivitypubMixin:
# there OUGHT to be only one match # there OUGHT to be only one match
return match.first() return match.first()
def broadcast(self, activity, sender, software=None, queue=MEDIUM): def broadcast(self, activity, sender, software=None, queue=BROADCAST):
"""send out an activity""" """send out an activity"""
broadcast_task.apply_async( broadcast_task.apply_async(
args=( args=(
@ -137,7 +138,7 @@ class ActivitypubMixin:
queue=queue, queue=queue,
) )
def get_recipients(self, software=None) -> List[str]: def get_recipients(self, software=None) -> list[str]:
"""figure out which inbox urls to post to""" """figure out which inbox urls to post to"""
# first we have to figure out who should receive this activity # first we have to figure out who should receive this activity
privacy = self.privacy if hasattr(self, "privacy") else "public" privacy = self.privacy if hasattr(self, "privacy") else "public"
@ -198,7 +199,14 @@ class ActivitypubMixin:
class ObjectMixin(ActivitypubMixin): class ObjectMixin(ActivitypubMixin):
"""add this mixin for object models that are AP serializable""" """add this mixin for object models that are AP serializable"""
def save(self, *args, created=None, software=None, priority=MEDIUM, **kwargs): def save(
self,
*args: Any,
created: Optional[bool] = None,
software: Any = None,
priority: str = BROADCAST,
**kwargs: Any,
) -> None:
"""broadcast created/updated/deleted objects as appropriate""" """broadcast created/updated/deleted objects as appropriate"""
broadcast = kwargs.get("broadcast", True) broadcast = kwargs.get("broadcast", True)
# this bonus kwarg would cause an error in the base save method # this bonus kwarg would cause an error in the base save method
@ -234,8 +242,8 @@ class ObjectMixin(ActivitypubMixin):
activity = self.to_create_activity(user) activity = self.to_create_activity(user)
self.broadcast(activity, user, software=software, queue=priority) self.broadcast(activity, user, software=software, queue=priority)
except AttributeError: except AttributeError:
# janky as heck, this catches the mutliple inheritence chain # janky as heck, this catches the multiple inheritance chain
# for boosts and ignores this auxilliary broadcast # for boosts and ignores this auxiliary broadcast
return return
return return
@ -311,7 +319,7 @@ class OrderedCollectionPageMixin(ObjectMixin):
@property @property
def collection_remote_id(self): def collection_remote_id(self):
"""this can be overriden if there's a special remote id, ie outbox""" """this can be overridden if there's a special remote id, ie outbox"""
return self.remote_id return self.remote_id
def to_ordered_collection( def to_ordered_collection(
@ -339,7 +347,7 @@ class OrderedCollectionPageMixin(ObjectMixin):
activity["id"] = remote_id activity["id"] = remote_id
paginated = Paginator(queryset, PAGE_LENGTH) paginated = Paginator(queryset, PAGE_LENGTH)
# add computed fields specific to orderd collections # add computed fields specific to ordered collections
activity["totalItems"] = paginated.count activity["totalItems"] = paginated.count
activity["first"] = f"{remote_id}?page=1" activity["first"] = f"{remote_id}?page=1"
activity["last"] = f"{remote_id}?page={paginated.num_pages}" activity["last"] = f"{remote_id}?page={paginated.num_pages}"
@ -379,7 +387,7 @@ class CollectionItemMixin(ActivitypubMixin):
activity_serializer = activitypub.CollectionItem activity_serializer = activitypub.CollectionItem
def broadcast(self, activity, sender, software="bookwyrm", queue=MEDIUM): def broadcast(self, activity, sender, software="bookwyrm", queue=BROADCAST):
"""only send book collection updates to other bookwyrm instances""" """only send book collection updates to other bookwyrm instances"""
super().broadcast(activity, sender, software=software, queue=queue) super().broadcast(activity, sender, software=software, queue=queue)
@ -400,12 +408,12 @@ class CollectionItemMixin(ActivitypubMixin):
return [] return []
return [collection_field.user] return [collection_field.user]
def save(self, *args, broadcast=True, priority=MEDIUM, **kwargs): def save(self, *args, broadcast=True, priority=BROADCAST, **kwargs):
"""broadcast updated""" """broadcast updated"""
# first off, we want to save normally no matter what # first off, we want to save normally no matter what
super().save(*args, **kwargs) super().save(*args, **kwargs)
# list items can be updateda, normally you would only broadcast on created # list items can be updated, normally you would only broadcast on created
if not broadcast or not self.user.local: if not broadcast or not self.user.local:
return return
@ -444,7 +452,7 @@ class CollectionItemMixin(ActivitypubMixin):
class ActivityMixin(ActivitypubMixin): class ActivityMixin(ActivitypubMixin):
"""add this mixin for models that are AP serializable""" """add this mixin for models that are AP serializable"""
def save(self, *args, broadcast=True, priority=MEDIUM, **kwargs): def save(self, *args, broadcast=True, priority=BROADCAST, **kwargs):
"""broadcast activity""" """broadcast activity"""
super().save(*args, **kwargs) super().save(*args, **kwargs)
user = self.user if hasattr(self, "user") else self.user_subject user = self.user if hasattr(self, "user") else self.user_subject
@ -506,15 +514,15 @@ def unfurl_related_field(related_field, sort_field=None):
return related_field.remote_id return related_field.remote_id
@app.task(queue=MEDIUM) @app.task(queue=BROADCAST)
def broadcast_task(sender_id: int, activity: str, recipients: List[str]): def broadcast_task(sender_id: int, activity: str, recipients: list[str]):
"""the celery task for broadcast""" """the celery task for broadcast"""
user_model = apps.get_model("bookwyrm.User", require_ready=True) user_model = apps.get_model("bookwyrm.User", require_ready=True)
sender = user_model.objects.select_related("key_pair").get(id=sender_id) sender = user_model.objects.select_related("key_pair").get(id=sender_id)
asyncio.run(async_broadcast(recipients, sender, activity)) asyncio.run(async_broadcast(recipients, sender, activity))
async def async_broadcast(recipients: List[str], sender, data: str): async def async_broadcast(recipients: list[str], sender, data: str):
"""Send all the broadcasts simultaneously""" """Send all the broadcasts simultaneously"""
timeout = aiohttp.ClientTimeout(total=10) timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(timeout=timeout) as session: async with aiohttp.ClientSession(timeout=timeout) as session:
@ -529,7 +537,7 @@ async def async_broadcast(recipients: List[str], sender, data: str):
async def sign_and_send( async def sign_and_send(
session: aiohttp.ClientSession, sender, data: str, destination: str session: aiohttp.ClientSession, sender, data: str, destination: str, **kwargs
): ):
"""Sign the messages and send them in an asynchronous bundle""" """Sign the messages and send them in an asynchronous bundle"""
now = http_date() now = http_date()
@ -539,11 +547,19 @@ async def sign_and_send(
raise ValueError("No private key found for sender") raise ValueError("No private key found for sender")
digest = make_digest(data) digest = make_digest(data)
signature = make_signature(
"post",
sender,
destination,
now,
digest=digest,
use_legacy_key=kwargs.get("use_legacy_key"),
)
headers = { headers = {
"Date": now, "Date": now,
"Digest": digest, "Digest": digest,
"Signature": make_signature(sender, destination, now, digest), "Signature": signature,
"Content-Type": "application/activity+json; charset=utf-8", "Content-Type": "application/activity+json; charset=utf-8",
"User-Agent": USER_AGENT, "User-Agent": USER_AGENT,
} }
@ -554,6 +570,14 @@ async def sign_and_send(
logger.exception( logger.exception(
"Failed to send broadcast to %s: %s", destination, response.reason "Failed to send broadcast to %s: %s", destination, response.reason
) )
if kwargs.get("use_legacy_key") is not True:
logger.info("Trying again with legacy keyId header value")
asyncio.ensure_future(
sign_and_send(
session, sender, data, destination, use_legacy_key=True
)
)
return response return response
except asyncio.TimeoutError: except asyncio.TimeoutError:
logger.info("Connection timed out for url: %s", destination) logger.info("Connection timed out for url: %s", destination)
@ -565,7 +589,7 @@ async def sign_and_send(
def to_ordered_collection_page( def to_ordered_collection_page(
queryset, remote_id, id_only=False, page=1, pure=False, **kwargs queryset, remote_id, id_only=False, page=1, pure=False, **kwargs
): ):
"""serialize and pagiante a queryset""" """serialize and paginate a queryset"""
paginated = Paginator(queryset, PAGE_LENGTH) paginated = Paginator(queryset, PAGE_LENGTH)
activity_page = paginated.get_page(page) activity_page = paginated.get_page(page)

View file

@ -24,7 +24,7 @@ class AnnualGoal(BookWyrmModel):
) )
class Meta: class Meta:
"""unqiueness constraint""" """uniqueness constraint"""
unique_together = ("user", "year") unique_together = ("user", "year")
@ -52,7 +52,7 @@ class AnnualGoal(BookWyrmModel):
user=self.user, user=self.user,
book__in=book_ids, book__in=book_ids,
) )
return {r.book.id: r.rating for r in reviews} return {r.book_id: r.rating for r in reviews}
@property @property
def progress(self): def progress(self):

View file

@ -8,7 +8,7 @@ from django.db import models, transaction
from django.db.models import Q from django.db.models import Q
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from bookwyrm.tasks import app, LOW from bookwyrm.tasks import app, MISC
from .base_model import BookWyrmModel from .base_model import BookWyrmModel
from .user import User from .user import User
@ -65,7 +65,7 @@ class AutoMod(AdminModel):
created_by = models.ForeignKey("User", on_delete=models.PROTECT) created_by = models.ForeignKey("User", on_delete=models.PROTECT)
@app.task(queue=LOW) @app.task(queue=MISC)
def automod_task(): def automod_task():
"""Create reports""" """Create reports"""
if not AutoMod.objects.exists(): if not AutoMod.objects.exists():

View file

@ -1,8 +1,8 @@
""" database schema for info about authors """ """ database schema for info about authors """
import re import re
from typing import Tuple, Any
from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.indexes import GinIndex
from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.db import models from django.db import models
from bookwyrm import activitypub from bookwyrm import activitypub
@ -24,6 +24,13 @@ class Author(BookDataModel):
gutenberg_id = fields.CharField( gutenberg_id = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True max_length=255, blank=True, null=True, deduplication_field=True
) )
isfdb = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
website = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
# idk probably other keys would be useful here? # idk probably other keys would be useful here?
born = fields.DateTimeField(blank=True, null=True) born = fields.DateTimeField(blank=True, null=True)
died = fields.DateTimeField(blank=True, null=True) died = fields.DateTimeField(blank=True, null=True)
@ -33,17 +40,8 @@ class Author(BookDataModel):
) )
bio = fields.HtmlField(null=True, blank=True) bio = fields.HtmlField(null=True, blank=True)
def save(self, *args, **kwargs): def save(self, *args: Tuple[Any, ...], **kwargs: dict[str, Any]) -> None:
"""clear related template caches""" """normalize isni format"""
# clear template caches
if self.id:
cache_keys = [
make_template_fragment_key("titleby", [book])
for book in self.book_set.values_list("id", flat=True)
]
cache.delete_many(cache_keys)
# normalize isni format
if self.isni: if self.isni:
self.isni = re.sub(r"\s", "", self.isni) self.isni = re.sub(r"\s", "", self.isni)
@ -60,6 +58,11 @@ class Author(BookDataModel):
"""generate the url from the openlibrary id""" """generate the url from the openlibrary id"""
return f"https://openlibrary.org/authors/{self.openlibrary_key}" return f"https://openlibrary.org/authors/{self.openlibrary_key}"
@property
def isfdb_link(self):
"""generate the url from the isni id"""
return f"https://www.isfdb.org/cgi-bin/ea.cgi?{self.isfdb}"
def get_remote_id(self): def get_remote_id(self):
"""editions and works both use "book" instead of model_name""" """editions and works both use "book" instead of model_name"""
return f"https://{DOMAIN}/author/{self.id}" return f"https://{DOMAIN}/author/{self.id}"

View file

@ -1,10 +1,11 @@
""" database schema for books and shelves """ """ database schema for books and shelves """
from itertools import chain
import re import re
from typing import Any
from django.contrib.postgres.search import SearchVectorField from django.contrib.postgres.search import SearchVectorField
from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.indexes import GinIndex
from django.core.cache import cache from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.db import models, transaction from django.db import models, transaction
from django.db.models import Prefetch from django.db.models import Prefetch
from django.dispatch import receiver from django.dispatch import receiver
@ -14,10 +15,12 @@ from model_utils.managers import InheritanceManager
from imagekit.models import ImageSpecField from imagekit.models import ImageSpecField
from bookwyrm import activitypub from bookwyrm import activitypub
from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator
from bookwyrm.preview_images import generate_edition_preview_image_task from bookwyrm.preview_images import generate_edition_preview_image_task
from bookwyrm.settings import ( from bookwyrm.settings import (
DOMAIN, DOMAIN,
DEFAULT_LANGUAGE, DEFAULT_LANGUAGE,
LANGUAGE_ARTICLES,
ENABLE_PREVIEW_IMAGES, ENABLE_PREVIEW_IMAGES,
ENABLE_THUMBNAIL_GENERATION, ENABLE_THUMBNAIL_GENERATION,
) )
@ -55,6 +58,12 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
asin = fields.CharField( asin = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True max_length=255, blank=True, null=True, deduplication_field=True
) )
aasin = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
isfdb = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
search_vector = SearchVectorField(null=True) search_vector = SearchVectorField(null=True)
last_edited_by = fields.ForeignKey( last_edited_by = fields.ForeignKey(
@ -73,12 +82,17 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
"""generate the url from the inventaire id""" """generate the url from the inventaire id"""
return f"https://inventaire.io/entity/{self.inventaire_id}" return f"https://inventaire.io/entity/{self.inventaire_id}"
@property
def isfdb_link(self):
"""generate the url from the isfdb id"""
return f"https://www.isfdb.org/cgi-bin/title.cgi?{self.isfdb}"
class Meta: class Meta:
"""can't initialize this model, that wouldn't make sense""" """can't initialize this model, that wouldn't make sense"""
abstract = True abstract = True
def save(self, *args, **kwargs): def save(self, *args: Any, **kwargs: Any) -> None:
"""ensure that the remote_id is within this instance""" """ensure that the remote_id is within this instance"""
if self.id: if self.id:
self.remote_id = self.get_remote_id() self.remote_id = self.get_remote_id()
@ -192,21 +206,24 @@ class Book(BookDataModel):
text += f" ({self.edition_info})" text += f" ({self.edition_info})"
return text return text
def save(self, *args, **kwargs): def save(self, *args: Any, **kwargs: Any) -> None:
"""can't be abstract for query reasons, but you shouldn't USE it""" """can't be abstract for query reasons, but you shouldn't USE it"""
if not isinstance(self, Edition) and not isinstance(self, Work): if not isinstance(self, Edition) and not isinstance(self, Work):
raise ValueError("Books should be added as Editions or Works") raise ValueError("Books should be added as Editions or Works")
# clear template caches
cache_key = make_template_fragment_key("titleby", [self.id])
cache.delete(cache_key)
return super().save(*args, **kwargs) return super().save(*args, **kwargs)
def get_remote_id(self): def get_remote_id(self):
"""editions and works both use "book" instead of model_name""" """editions and works both use "book" instead of model_name"""
return f"https://{DOMAIN}/book/{self.id}" return f"https://{DOMAIN}/book/{self.id}"
def guess_sort_title(self):
"""Get a best-guess sort title for the current book"""
articles = chain(
*(LANGUAGE_ARTICLES.get(language, ()) for language in tuple(self.languages))
)
return re.sub(f'^{" |^".join(articles)} ', "", str(self.title).lower())
def __repr__(self): def __repr__(self):
# pylint: disable=consider-using-f-string # pylint: disable=consider-using-f-string
return "<{} key={!r} title={!r}>".format( return "<{} key={!r} title={!r}>".format(
@ -312,10 +329,15 @@ class Edition(Book):
serialize_reverse_fields = [("file_links", "fileLinks", "-created_date")] serialize_reverse_fields = [("file_links", "fileLinks", "-created_date")]
deserialize_reverse_fields = [("file_links", "fileLinks")] deserialize_reverse_fields = [("file_links", "fileLinks")]
@property
def hyphenated_isbn13(self):
"""generate the hyphenated version of the ISBN-13"""
return hyphenator.hyphenate(self.isbn_13)
def get_rank(self): def get_rank(self):
"""calculate how complete the data is on this edition""" """calculate how complete the data is on this edition"""
rank = 0 rank = 0
# big ups for havinga cover # big ups for having a cover
rank += int(bool(self.cover)) * 3 rank += int(bool(self.cover)) * 3
# is it in the instance's preferred language? # is it in the instance's preferred language?
rank += int(bool(DEFAULT_LANGUAGE in self.languages)) rank += int(bool(DEFAULT_LANGUAGE in self.languages))
@ -335,7 +357,7 @@ class Edition(Book):
# max rank is 9 # max rank is 9
return rank return rank
def save(self, *args, **kwargs): def save(self, *args: Any, **kwargs: Any) -> None:
"""set some fields on the edition object""" """set some fields on the edition object"""
# calculate isbn 10/13 # calculate isbn 10/13
if self.isbn_13 and self.isbn_13[:3] == "978" and not self.isbn_10: if self.isbn_13 and self.isbn_13[:3] == "978" and not self.isbn_10:
@ -357,8 +379,25 @@ class Edition(Book):
for author_id in self.authors.values_list("id", flat=True): for author_id in self.authors.values_list("id", flat=True):
cache.delete(f"author-books-{author_id}") cache.delete(f"author-books-{author_id}")
# Create sort title by removing articles from title
if self.sort_title in [None, ""]:
self.sort_title = self.guess_sort_title()
return super().save(*args, **kwargs) return super().save(*args, **kwargs)
@transaction.atomic
def repair(self):
"""If an edition is in a bad state (missing a work), let's fix that"""
# made sure it actually NEEDS reapir
if self.parent_work:
return
new_work = Work.objects.create(title=self.title)
new_work.authors.set(self.authors.all())
self.parent_work = new_work
self.save(update_fields=["parent_work"], broadcast=False)
@classmethod @classmethod
def viewer_aware_objects(cls, viewer): def viewer_aware_objects(cls, viewer):
"""annotate a book query with metadata related to the user""" """annotate a book query with metadata related to the user"""

View file

@ -20,8 +20,9 @@ class Favorite(ActivityMixin, BookWyrmModel):
activity_serializer = activitypub.Like activity_serializer = activitypub.Like
# pylint: disable=unused-argument
@classmethod @classmethod
def ignore_activity(cls, activity): def ignore_activity(cls, activity, allow_external_connections=True):
"""don't bother with incoming favs of unknown statuses""" """don't bother with incoming favs of unknown statuses"""
return not Status.objects.filter(remote_id=activity.object).exists() return not Status.objects.filter(remote_id=activity.object).exists()

View file

@ -61,7 +61,7 @@ class FederatedServer(BookWyrmModel):
).update(active=True, deactivation_reason=None) ).update(active=True, deactivation_reason=None)
@classmethod @classmethod
def is_blocked(cls, url): def is_blocked(cls, url: str) -> bool:
"""look up if a domain is blocked""" """look up if a domain is blocked"""
url = urlparse(url) url = urlparse(url)
domain = url.netloc domain = url.netloc

View file

@ -1,5 +1,6 @@
""" activitypub-aware django model fields """ """ activitypub-aware django model fields """
from dataclasses import MISSING from dataclasses import MISSING
from datetime import datetime
import re import re
from uuid import uuid4 from uuid import uuid4
from urllib.parse import urljoin from urllib.parse import urljoin
@ -7,12 +8,14 @@ from urllib.parse import urljoin
import dateutil.parser import dateutil.parser
from dateutil.parser import ParserError from dateutil.parser import ParserError
from django.contrib.postgres.fields import ArrayField as DjangoArrayField from django.contrib.postgres.fields import ArrayField as DjangoArrayField
from django.contrib.postgres.fields import CICharField as DjangoCICharField
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.db import models from django.db import models
from django.forms import ClearableFileInput, ImageField as DjangoImageField from django.forms import ClearableFileInput, ImageField as DjangoImageField
from django.utils import timezone from django.utils import timezone
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from django.utils.encoding import filepath_to_uri from django.utils.encoding import filepath_to_uri
from markdown import markdown
from bookwyrm import activitypub from bookwyrm import activitypub
from bookwyrm.connectors import get_image from bookwyrm.connectors import get_image
@ -66,16 +69,20 @@ class ActivitypubFieldMixin:
self.activitypub_field = activitypub_field self.activitypub_field = activitypub_field
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def set_field_from_activity(self, instance, data, overwrite=True): def set_field_from_activity(
"""helper function for assinging a value to the field. Returns if changed""" self, instance, data, overwrite=True, allow_external_connections=True
):
"""helper function for assigning a value to the field. Returns if changed"""
try: try:
value = getattr(data, self.get_activitypub_field()) value = getattr(data, self.get_activitypub_field())
except AttributeError: except AttributeError:
# masssively hack-y workaround for boosts # massively hack-y workaround for boosts
if self.get_activitypub_field() != "attributedTo": if self.get_activitypub_field() != "attributedTo":
raise raise
value = getattr(data, "actor") value = getattr(data, "actor")
formatted = self.field_from_activity(value) formatted = self.field_from_activity(
value, allow_external_connections=allow_external_connections
)
if formatted is None or formatted is MISSING or formatted == {}: if formatted is None or formatted is MISSING or formatted == {}:
return False return False
@ -115,7 +122,8 @@ class ActivitypubFieldMixin:
return {self.activitypub_wrapper: value} return {self.activitypub_wrapper: value}
return value return value
def field_from_activity(self, value): # pylint: disable=unused-argument
def field_from_activity(self, value, allow_external_connections=True):
"""formatter to convert activitypub into a model value""" """formatter to convert activitypub into a model value"""
if value and hasattr(self, "activitypub_wrapper"): if value and hasattr(self, "activitypub_wrapper"):
value = value.get(self.activitypub_wrapper) value = value.get(self.activitypub_wrapper)
@ -137,7 +145,7 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
self.load_remote = load_remote self.load_remote = load_remote
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def field_from_activity(self, value): def field_from_activity(self, value, allow_external_connections=True):
if not value: if not value:
return None return None
@ -158,7 +166,11 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
if not self.load_remote: if not self.load_remote:
# only look in the local database # only look in the local database
return related_model.find_existing_by_remote_id(value) return related_model.find_existing_by_remote_id(value)
return activitypub.resolve_remote_id(value, model=related_model) return activitypub.resolve_remote_id(
value,
model=related_model,
allow_external_connections=allow_external_connections,
)
class RemoteIdField(ActivitypubFieldMixin, models.CharField): class RemoteIdField(ActivitypubFieldMixin, models.CharField):
@ -210,7 +222,7 @@ PrivacyLevels = [
class PrivacyField(ActivitypubFieldMixin, models.CharField): class PrivacyField(ActivitypubFieldMixin, models.CharField):
"""this maps to two differente activitypub fields""" """this maps to two different activitypub fields"""
public = "https://www.w3.org/ns/activitystreams#Public" public = "https://www.w3.org/ns/activitystreams#Public"
@ -218,7 +230,9 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
super().__init__(*args, max_length=255, choices=PrivacyLevels, default="public") super().__init__(*args, max_length=255, choices=PrivacyLevels, default="public")
# pylint: disable=invalid-name # pylint: disable=invalid-name
def set_field_from_activity(self, instance, data, overwrite=True): def set_field_from_activity(
self, instance, data, overwrite=True, allow_external_connections=True
):
if not overwrite: if not overwrite:
return False return False
@ -233,7 +247,11 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
break break
if not user_field: if not user_field:
raise ValidationError("No user field found for privacy", data) raise ValidationError("No user field found for privacy", data)
user = activitypub.resolve_remote_id(getattr(data, user_field), model="User") user = activitypub.resolve_remote_id(
getattr(data, user_field),
model="User",
allow_external_connections=allow_external_connections,
)
if to == [self.public]: if to == [self.public]:
setattr(instance, self.name, "public") setattr(instance, self.name, "public")
@ -294,13 +312,17 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
self.link_only = link_only self.link_only = link_only
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def set_field_from_activity(self, instance, data, overwrite=True): def set_field_from_activity(
self, instance, data, overwrite=True, allow_external_connections=True
):
"""helper function for assigning a value to the field""" """helper function for assigning a value to the field"""
if not overwrite and getattr(instance, self.name).exists(): if not overwrite and getattr(instance, self.name).exists():
return False return False
value = getattr(data, self.get_activitypub_field()) value = getattr(data, self.get_activitypub_field())
formatted = self.field_from_activity(value) formatted = self.field_from_activity(
value, allow_external_connections=allow_external_connections
)
if formatted is None or formatted is MISSING: if formatted is None or formatted is MISSING:
return False return False
getattr(instance, self.name).set(formatted) getattr(instance, self.name).set(formatted)
@ -312,7 +334,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
return f"{value.instance.remote_id}/{self.name}" return f"{value.instance.remote_id}/{self.name}"
return [i.remote_id for i in value.all()] return [i.remote_id for i in value.all()]
def field_from_activity(self, value): def field_from_activity(self, value, allow_external_connections=True):
if value is None or value is MISSING: if value is None or value is MISSING:
return None return None
if not isinstance(value, list): if not isinstance(value, list):
@ -325,7 +347,11 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
except ValidationError: except ValidationError:
continue continue
items.append( items.append(
activitypub.resolve_remote_id(remote_id, model=self.related_model) activitypub.resolve_remote_id(
remote_id,
model=self.related_model,
allow_external_connections=allow_external_connections,
)
) )
return items return items
@ -343,18 +369,29 @@ class TagField(ManyToManyField):
activity_type = item.__class__.__name__ activity_type = item.__class__.__name__
if activity_type == "User": if activity_type == "User":
activity_type = "Mention" activity_type = "Mention"
if activity_type == "Hashtag":
name = item.name
else:
name = f"@{getattr(item, item.name_field)}"
tags.append( tags.append(
activitypub.Link( activitypub.Link(
href=item.remote_id, href=item.remote_id,
name=getattr(item, item.name_field), name=name,
type=activity_type, type=activity_type,
) )
) )
return tags return tags
def field_from_activity(self, value): def field_from_activity(self, value, allow_external_connections=True):
if not isinstance(value, list): if not isinstance(value, list):
return None # GoToSocial DMs and single-user mentions are
# sent as objects, not as an array of objects
if isinstance(value, dict):
value = [value]
else:
return None
items = [] items = []
for link_json in value: for link_json in value:
link = activitypub.Link(**link_json) link = activitypub.Link(**link_json)
@ -364,9 +401,22 @@ class TagField(ManyToManyField):
if tag_type != self.related_model.activity_serializer.type: if tag_type != self.related_model.activity_serializer.type:
# tags can contain multiple types # tags can contain multiple types
continue continue
items.append(
activitypub.resolve_remote_id(link.href, model=self.related_model) if tag_type == "Hashtag":
) # we already have all data to create hashtags,
# no need to fetch from remote
item = self.related_model.activity_serializer(**link_json)
hashtag = item.to_model(model=self.related_model, save=True)
items.append(hashtag)
else:
# for other tag types we fetch them remotely
items.append(
activitypub.resolve_remote_id(
link.href,
model=self.related_model,
allow_external_connections=allow_external_connections,
)
)
return items return items
@ -389,11 +439,15 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
self.alt_field = alt_field self.alt_field = alt_field
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
# pylint: disable=arguments-differ,arguments-renamed # pylint: disable=arguments-differ,arguments-renamed,too-many-arguments
def set_field_from_activity(self, instance, data, save=True, overwrite=True): def set_field_from_activity(
"""helper function for assinging a value to the field""" self, instance, data, save=True, overwrite=True, allow_external_connections=True
):
"""helper function for assigning a value to the field"""
value = getattr(data, self.get_activitypub_field()) value = getattr(data, self.get_activitypub_field())
formatted = self.field_from_activity(value) formatted = self.field_from_activity(
value, allow_external_connections=allow_external_connections
)
if formatted is None or formatted is MISSING: if formatted is None or formatted is MISSING:
return False return False
@ -425,7 +479,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
return activitypub.Document(url=url, name=alt) return activitypub.Document(url=url, name=alt)
def field_from_activity(self, value): def field_from_activity(self, value, allow_external_connections=True):
image_slug = value image_slug = value
# when it's an inline image (User avatar/icon, Book cover), it's a json # when it's an inline image (User avatar/icon, Book cover), it's a json
# blob, but when it's an attached image, it's just a url # blob, but when it's an attached image, it's just a url
@ -480,9 +534,11 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
return None return None
return value.isoformat() return value.isoformat()
def field_from_activity(self, value): def field_from_activity(self, value, allow_external_connections=True):
missing_fields = datetime(1970, 1, 1) # "2022-10" => "2022-10-01"
try: try:
date_value = dateutil.parser.parse(value) # TODO(dato): investigate `ignoretz=True` wrt bookwyrm#3028.
date_value = dateutil.parser.parse(value, default=missing_fields)
try: try:
return timezone.make_aware(date_value) return timezone.make_aware(date_value)
except ValueError: except ValueError:
@ -494,11 +550,14 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
class HtmlField(ActivitypubFieldMixin, models.TextField): class HtmlField(ActivitypubFieldMixin, models.TextField):
"""a text field for storing html""" """a text field for storing html"""
def field_from_activity(self, value): def field_from_activity(self, value, allow_external_connections=True):
if not value or value == MISSING: if not value or value == MISSING:
return None return None
return clean(value) return clean(value)
def field_to_activity(self, value):
return markdown(value) if value else value
class ArrayField(ActivitypubFieldMixin, DjangoArrayField): class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
"""activitypub-aware array field""" """activitypub-aware array field"""
@ -511,6 +570,10 @@ class CharField(ActivitypubFieldMixin, models.CharField):
"""activitypub-aware char field""" """activitypub-aware char field"""
class CICharField(ActivitypubFieldMixin, DjangoCICharField):
"""activitypub-aware cichar field"""
class URLField(ActivitypubFieldMixin, models.URLField): class URLField(ActivitypubFieldMixin, models.URLField):
"""activitypub-aware url field""" """activitypub-aware url field"""

View file

@ -0,0 +1,23 @@
""" model for tags """
from bookwyrm import activitypub
from .activitypub_mixin import ActivitypubMixin
from .base_model import BookWyrmModel
from .fields import CICharField
class Hashtag(ActivitypubMixin, BookWyrmModel):
"a hashtag which can be used in statuses"
name = CICharField(
max_length=256,
blank=False,
null=False,
activitypub_field="name",
deduplication_field=True,
)
name_field = "name"
activity_serializer = activitypub.Hashtag
def __repr__(self):
return f"<{self.__class__} id={self.id} name={self.name}>"

View file

@ -1,4 +1,5 @@
""" track progress of goodreads imports """ """ track progress of goodreads imports """
from datetime import datetime
import math import math
import re import re
import dateutil.parser import dateutil.parser
@ -19,7 +20,7 @@ from bookwyrm.models import (
Review, Review,
ReviewRating, ReviewRating,
) )
from bookwyrm.tasks import app, LOW from bookwyrm.tasks import app, IMPORT_TRIGGERED, IMPORTS
from .fields import PrivacyLevels from .fields import PrivacyLevels
@ -54,10 +55,10 @@ ImportStatuses = [
class ImportJob(models.Model): class ImportJob(models.Model):
"""entry for a specific request for book data import""" """entry for a specific request for book data import"""
user = models.ForeignKey(User, on_delete=models.CASCADE) user: User = models.ForeignKey(User, on_delete=models.CASCADE)
created_date = models.DateTimeField(default=timezone.now) created_date = models.DateTimeField(default=timezone.now)
updated_date = models.DateTimeField(default=timezone.now) updated_date = models.DateTimeField(default=timezone.now)
include_reviews = models.BooleanField(default=True) include_reviews: bool = models.BooleanField(default=True)
mappings = models.JSONField() mappings = models.JSONField()
source = models.CharField(max_length=100) source = models.CharField(max_length=100)
privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels) privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels)
@ -74,10 +75,9 @@ class ImportJob(models.Model):
task = start_import_task.delay(self.id) task = start_import_task.delay(self.id)
self.task_id = task.id self.task_id = task.id
self.status = "active" self.save(update_fields=["task_id"])
self.save(update_fields=["status", "task_id"])
def complete_job(self): def complete_job(self) -> None:
"""Report that the job has completed""" """Report that the job has completed"""
self.status = "complete" self.status = "complete"
self.complete = True self.complete = True
@ -253,42 +253,37 @@ class ImportItem(models.Model):
@property @property
def rating(self): def rating(self):
"""x/5 star rating for a book""" """x/5 star rating for a book"""
if self.normalized_data.get("rating"): if not self.normalized_data.get("rating"):
return None
try:
return float(self.normalized_data.get("rating")) return float(self.normalized_data.get("rating"))
return None except ValueError:
return None
def _parse_datefield(self, field, /):
if not (date := self.normalized_data.get(field)):
return None
defaults = datetime(1970, 1, 1) # "2022-10" => "2022-10-01"
parsed = dateutil.parser.parse(date, default=defaults)
# Keep timezone if import already had one, else use default.
return parsed if timezone.is_aware(parsed) else timezone.make_aware(parsed)
@property @property
def date_added(self): def date_added(self):
"""when the book was added to this dataset""" """when the book was added to this dataset"""
if self.normalized_data.get("date_added"): return self._parse_datefield("date_added")
parsed_date_added = dateutil.parser.parse(
self.normalized_data.get("date_added")
)
if timezone.is_aware(parsed_date_added):
# Keep timezone if import already had one
return parsed_date_added
return timezone.make_aware(parsed_date_added)
return None
@property @property
def date_started(self): def date_started(self):
"""when the book was started""" """when the book was started"""
if self.normalized_data.get("date_started"): return self._parse_datefield("date_started")
return timezone.make_aware(
dateutil.parser.parse(self.normalized_data.get("date_started"))
)
return None
@property @property
def date_read(self): def date_read(self):
"""the date a book was completed""" """the date a book was completed"""
if self.normalized_data.get("date_finished"): return self._parse_datefield("date_finished")
return timezone.make_aware(
dateutil.parser.parse(self.normalized_data.get("date_finished"))
)
return None
@property @property
def reads(self): def reads(self):
@ -328,10 +323,12 @@ class ImportItem(models.Model):
) )
@app.task(queue=LOW) @app.task(queue=IMPORTS)
def start_import_task(job_id): def start_import_task(job_id):
"""trigger the child tasks for each row""" """trigger the child tasks for each row"""
job = ImportJob.objects.get(id=job_id) job = ImportJob.objects.get(id=job_id)
job.status = "active"
job.save(update_fields=["status"])
# don't start the job if it was stopped from the UI # don't start the job if it was stopped from the UI
if job.complete: if job.complete:
return return
@ -345,7 +342,7 @@ def start_import_task(job_id):
job.save() job.save()
@app.task(queue=LOW) @app.task(queue=IMPORTS)
def import_item_task(item_id): def import_item_task(item_id):
"""resolve a row into a book""" """resolve a row into a book"""
item = ImportItem.objects.get(id=item_id) item = ImportItem.objects.get(id=item_id)
@ -395,7 +392,7 @@ def handle_imported_book(item):
shelved_date = item.date_added or timezone.now() shelved_date = item.date_added or timezone.now()
ShelfBook( ShelfBook(
book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date
).save(priority=LOW) ).save(priority=IMPORT_TRIGGERED)
for read in item.reads: for read in item.reads:
# check for an existing readthrough with the same dates # check for an existing readthrough with the same dates
@ -437,7 +434,7 @@ def handle_imported_book(item):
published_date=published_date_guess, published_date=published_date_guess,
privacy=job.privacy, privacy=job.privacy,
) )
review.save(software="bookwyrm", priority=LOW) review.save(software="bookwyrm", priority=IMPORT_TRIGGERED)
else: else:
# just a rating # just a rating
review = ReviewRating.objects.filter( review = ReviewRating.objects.filter(
@ -454,7 +451,7 @@ def handle_imported_book(item):
published_date=published_date_guess, published_date=published_date_guess,
privacy=job.privacy, privacy=job.privacy,
) )
review.save(software="bookwyrm", priority=LOW) review.save(software="bookwyrm", priority=IMPORT_TRIGGERED)
# only broadcast this review to other bookwyrm instances # only broadcast this review to other bookwyrm instances
item.linked_review = review item.linked_review = review

View file

@ -31,7 +31,7 @@ class Link(ActivitypubMixin, BookWyrmModel):
@property @property
def name(self): def name(self):
"""link name via the assocaited domain""" """link name via the associated domain"""
return self.domain.name return self.domain.name
def save(self, *args, **kwargs): def save(self, *args, **kwargs):

View file

@ -2,8 +2,8 @@
from django.db import models, transaction from django.db import models, transaction
from django.dispatch import receiver from django.dispatch import receiver
from .base_model import BookWyrmModel from .base_model import BookWyrmModel
from . import Boost, Favorite, GroupMemberInvitation, ImportJob, ListItem, Report from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain
from . import Status, User, UserFollowRequest from . import ListItem, Report, Status, User, UserFollowRequest
class Notification(BookWyrmModel): class Notification(BookWyrmModel):
@ -28,6 +28,7 @@ class Notification(BookWyrmModel):
# Admin # Admin
REPORT = "REPORT" REPORT = "REPORT"
LINK_DOMAIN = "LINK_DOMAIN"
# Groups # Groups
INVITE = "INVITE" INVITE = "INVITE"
@ -43,7 +44,7 @@ class Notification(BookWyrmModel):
NotificationType = models.TextChoices( NotificationType = models.TextChoices(
# there has got be a better way to do this # there has got be a better way to do this
"NotificationType", "NotificationType",
f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}", f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
) )
user = models.ForeignKey("User", on_delete=models.CASCADE) user = models.ForeignKey("User", on_delete=models.CASCADE)
@ -64,6 +65,7 @@ class Notification(BookWyrmModel):
"ListItem", symmetrical=False, related_name="notifications" "ListItem", symmetrical=False, related_name="notifications"
) )
related_reports = models.ManyToManyField("Report", symmetrical=False) related_reports = models.ManyToManyField("Report", symmetrical=False)
related_link_domains = models.ManyToManyField("LinkDomain", symmetrical=False)
@classmethod @classmethod
@transaction.atomic @transaction.atomic
@ -241,6 +243,26 @@ def notify_admins_on_report(sender, instance, created, *args, **kwargs):
notification.related_reports.add(instance) notification.related_reports.add(instance)
@receiver(models.signals.post_save, sender=LinkDomain)
@transaction.atomic
# pylint: disable=unused-argument
def notify_admins_on_link_domain(sender, instance, created, *args, **kwargs):
"""a new link domain needs to be verified"""
if not created:
# otherwise you'll get a notification when you approve a domain
return
# moderators and superusers should be notified
admins = User.admins()
for admin in admins:
notification, _ = Notification.objects.get_or_create(
user=admin,
notification_type=Notification.LINK_DOMAIN,
read=False,
)
notification.related_link_domains.add(instance)
@receiver(models.signals.post_save, sender=GroupMemberInvitation) @receiver(models.signals.post_save, sender=GroupMemberInvitation)
# pylint: disable=unused-argument # pylint: disable=unused-argument
def notify_user_on_group_invite(sender, instance, *args, **kwargs): def notify_user_on_group_invite(sender, instance, *args, **kwargs):
@ -262,7 +284,7 @@ def notify_user_on_list_item_add(sender, instance, created, *args, **kwargs):
return return
list_owner = instance.book_list.user list_owner = instance.book_list.user
# create a notification if somoene ELSE added to a local user's list # create a notification if someone ELSE added to a local user's list
if list_owner.local and list_owner != instance.user: if list_owner.local and list_owner != instance.user:
# keep the related_user singular, group the items # keep the related_user singular, group the items
Notification.notify_list_item(list_owner, instance) Notification.notify_list_item(list_owner, instance)

Some files were not shown because too many files have changed in this diff Show more