diff --git a/.dockerignore b/.dockerignore
index a5130c8bd..5edf3de0d 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -5,3 +5,4 @@ __pycache__
.git
.github
.pytest*
+.env
diff --git a/.env.example b/.env.example
index af1d6430c..fb0f7308d 100644
--- a/.env.example
+++ b/.env.example
@@ -8,7 +8,7 @@ USE_HTTPS=true
DOMAIN=your.domain.here
EMAIL=your@email.here
-# Instance defualt language (see options at bookwyrm/settings.py "LANGUAGES"
+# Instance default language (see options at bookwyrm/settings.py "LANGUAGES"
LANGUAGE_CODE="en-us"
# Used for deciding which editions to prefer
DEFAULT_LANGUAGE="English"
@@ -21,8 +21,8 @@ MEDIA_ROOT=images/
# Database configuration
PGPORT=5432
POSTGRES_PASSWORD=securedbypassword123
-POSTGRES_USER=fedireads
-POSTGRES_DB=fedireads
+POSTGRES_USER=bookwyrm
+POSTGRES_DB=bookwyrm
POSTGRES_HOST=db
# Redis activity stream manager
@@ -32,12 +32,17 @@ REDIS_ACTIVITY_PORT=6379
REDIS_ACTIVITY_PASSWORD=redispassword345
# Optional, use a different redis database (defaults to 0)
# REDIS_ACTIVITY_DB_INDEX=0
+# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
+# REDIS_ACTIVITY_URL=
# Redis as celery broker
+REDIS_BROKER_HOST=redis_broker
REDIS_BROKER_PORT=6379
REDIS_BROKER_PASSWORD=redispassword123
# Optional, use a different redis database (defaults to 0)
# REDIS_BROKER_DB_INDEX=0
+# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
+# REDIS_BROKER_URL=
# Monitoring for celery
FLOWER_PORT=8888
@@ -60,7 +65,7 @@ SEARCH_TIMEOUT=5
QUERY_TIMEOUT=5
# Thumbnails Generation
-ENABLE_THUMBNAIL_GENERATION=false
+ENABLE_THUMBNAIL_GENERATION=true
# S3 configuration
USE_S3=false
@@ -77,9 +82,15 @@ AWS_SECRET_ACCESS_KEY=
# AWS_S3_REGION_NAME=None # "fr-par"
# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud"
+# Commented are example values if you use Azure Blob Storage
+# USE_AZURE=true
+# AZURE_ACCOUNT_NAME= # "example-account-name"
+# AZURE_ACCOUNT_KEY= # "base64-encoded-access-key"
+# AZURE_CONTAINER= # "example-blob-container-name"
+# AZURE_CUSTOM_DOMAIN= # "example-account-name.blob.core.windows.net"
# Preview image generation can be computing and storage intensive
-# ENABLE_PREVIEW_IMAGES=True
+ENABLE_PREVIEW_IMAGES=False
# Specify RGB tuple or RGB hex strings,
# or use_dominant_color_light / use_dominant_color_dark
@@ -108,3 +119,21 @@ OTEL_EXPORTER_OTLP_ENDPOINT=
OTEL_EXPORTER_OTLP_HEADERS=
# Service name to identify your app
OTEL_SERVICE_NAME=
+
+# Set HTTP_X_FORWARDED_PROTO ONLY to true if you know what you are doing.
+# Only use it if your proxy is "swallowing" if the original request was made
+# via https. Please refer to the Django-Documentation and assess the risks
+# for your instance:
+# https://docs.djangoproject.com/en/3.2/ref/settings/#secure-proxy-ssl-header
+HTTP_X_FORWARDED_PROTO=false
+
+# TOTP settings
+# TWO_FACTOR_LOGIN_VALIDITY_WINDOW sets the number of codes either side
+# which will be accepted.
+TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
+TWO_FACTOR_LOGIN_MAX_SECONDS=60
+
+# Additional hosts to allow in the Content-Security-Policy, "self" (should be DOMAIN)
+# and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
+# Value should be a comma-separated list of host names.
+CSP_ADDITIONAL_HOSTS=
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
index 7258b6087..4e7be4af3 100644
--- a/.github/workflows/black.yml
+++ b/.github/workflows/black.yml
@@ -10,6 +10,8 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- - uses: psf/black@21.4b2
+ - uses: actions/checkout@v3
+ - uses: actions/setup-python@v4
+ - uses: psf/black@22.12.0
+ with:
+ version: 22.12.0
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index d35f90eb5..68bb05d7e 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -36,11 +36,11 @@ jobs:
steps:
- name: Checkout repository
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
- uses: github/codeql-action/init@v1
+ uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
- uses: github/codeql-action/autobuild@v1
+ uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@v1
+ uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/curlylint.yaml b/.github/workflows/curlylint.yaml
index 593a42837..8d5c6b4f7 100644
--- a/.github/workflows/curlylint.yaml
+++ b/.github/workflows/curlylint.yaml
@@ -10,7 +10,7 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Install curlylint
run: pip install curlylint
diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml
index 97a744813..da11fe09e 100644
--- a/.github/workflows/django-tests.yml
+++ b/.github/workflows/django-tests.yml
@@ -23,9 +23,9 @@ jobs:
ports:
- 5432:5432
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up Python
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install Dependencies
@@ -56,5 +56,6 @@ jobs:
EMAIL_USE_TLS: true
ENABLE_PREVIEW_IMAGES: false
ENABLE_THUMBNAIL_GENERATION: true
+ HTTP_X_FORWARDED_PROTO: false
run: |
pytest -n 3
diff --git a/.github/workflows/lint-frontend.yaml b/.github/workflows/lint-frontend.yaml
index c97ee02ad..0d0559e40 100644
--- a/.github/workflows/lint-frontend.yaml
+++ b/.github/workflows/lint-frontend.yaml
@@ -19,7 +19,7 @@ jobs:
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Install modules
run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
new file mode 100644
index 000000000..1a641edd2
--- /dev/null
+++ b/.github/workflows/mypy.yml
@@ -0,0 +1,50 @@
+name: Mypy
+
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+ branches: [ main ]
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python 3.9
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.9
+ - name: Install Dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r requirements.txt
+ - name: Analysing the code with mypy
+ env:
+ SECRET_KEY: beepbeep
+ DEBUG: false
+ USE_HTTPS: true
+ DOMAIN: your.domain.here
+ BOOKWYRM_DATABASE_BACKEND: postgres
+ MEDIA_ROOT: images/
+ POSTGRES_PASSWORD: hunter2
+ POSTGRES_USER: postgres
+ POSTGRES_DB: github_actions
+ POSTGRES_HOST: 127.0.0.1
+ CELERY_BROKER: ""
+ REDIS_BROKER_PORT: 6379
+ REDIS_BROKER_PASSWORD: beep
+ USE_DUMMY_CACHE: true
+ FLOWER_PORT: 8888
+ EMAIL_HOST: "smtp.mailgun.org"
+ EMAIL_PORT: 587
+ EMAIL_HOST_USER: ""
+ EMAIL_HOST_PASSWORD: ""
+ EMAIL_USE_TLS: true
+ ENABLE_PREVIEW_IMAGES: false
+ ENABLE_THUMBNAIL_GENERATION: true
+ HTTP_X_FORWARDED_PROTO: false
+ run: |
+ mypy bookwyrm celerywyrm
diff --git a/.github/workflows/prettier.yaml b/.github/workflows/prettier.yaml
index c4a031dba..501516ae1 100644
--- a/.github/workflows/prettier.yaml
+++ b/.github/workflows/prettier.yaml
@@ -14,10 +14,10 @@ jobs:
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Install modules
- run: npm install prettier
+ run: npm install prettier@2.5.1
- name: Run Prettier
run: npx prettier --check bookwyrm/static/js/*.js
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
index a3117f7cb..3811c97d3 100644
--- a/.github/workflows/pylint.yml
+++ b/.github/workflows/pylint.yml
@@ -12,9 +12,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up Python 3.9
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install Dependencies
diff --git a/.prettierrc b/.prettierrc
new file mode 100644
index 000000000..ed29060e6
--- /dev/null
+++ b/.prettierrc
@@ -0,0 +1 @@
+'trailingComma': 'es5'
\ No newline at end of file
diff --git a/FEDERATION.md b/FEDERATION.md
new file mode 100644
index 000000000..dd0c917e2
--- /dev/null
+++ b/FEDERATION.md
@@ -0,0 +1,333 @@
+# Federation
+
+BookWyrm uses the [ActivityPub](http://activitypub.rocks/) protocol to send and receive user activity between other BookWyrm instances and other services that implement ActivityPub. To handle book data, BookWyrm has a handful of extended Activity types which are not part of the standard, but are legible to other BookWyrm instances.
+
+## Activities and Objects
+
+### Users and relationships
+User relationship interactions follow the standard ActivityPub spec.
+
+- `Follow`: request to receive statuses from a user, and view their statuses that have followers-only privacy
+- `Accept`: approves a `Follow` and finalizes the relationship
+- `Reject`: denies a `Follow`
+- `Block`: prevent users from seeing one another's statuses, and prevents the blocked user from viewing the actor's profile
+- `Update`: updates a user's profile and settings
+- `Delete`: deactivates a user
+- `Undo`: reverses a `Follow` or `Block`
+
+### Activities
+- `Create/Status`: saves a new status in the database.
+- `Delete/Status`: Removes a status
+- `Like/Status`: Creates a favorite on the status
+- `Announce/Status`: Boosts the status into the actor's timeline
+- `Undo/*`,: Reverses a `Like` or `Announce`
+
+### Collections
+User's books and lists are represented by [`OrderedCollection`](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection)
+
+### Statuses
+
+BookWyrm is focused on book reading activities - it is not a general-purpose messaging application. For this reason, BookWyrm only accepts status `Create` activities if they are:
+
+- Direct messages (i.e., `Note`s with the privacy level `direct`, which mention a local user),
+- Related to a book (of a custom status type that includes the field `inReplyToBook`),
+- Replies to existing statuses saved in the database
+
+All other statuses will be received by the instance inbox, but by design **will not be delivered to user inboxes or displayed to users**.
+
+### Custom Object types
+
+With the exception of `Note`, the following object types are used in Bookwyrm but are not currently provided with a custom JSON-LD `@context` extension IRI. This is likely to change in future to make them true deserialisable JSON-LD objects.
+
+##### Note
+
+Within BookWyrm a `Note` is constructed according to [the ActivityStreams vocabulary](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-note), however `Note`s can only be created as direct messages or as replies to other statuses. As mentioned above, this also applies to incoming `Note`s.
+
+##### Review
+
+A `Review` is a status in response to a book (indicated by the `inReplyToBook` field), which has a title, body, and numerical rating between 0 (not rated) and 5.
+
+Example:
+
+```json
+{
+ "id": "https://example.net/user/library_lurker/review/2",
+ "type": "Review",
+ "published": "2023-06-30T21:43:46.013132+00:00",
+ "attributedTo": "https://example.net/user/library_lurker",
+ "content": "
This is an enjoyable book with great characters.
",
+ "to": ["https://example.net/user/library_lurker/followers"],
+ "cc": [],
+ "replies": {
+ "id": "https://example.net/user/library_lurker/review/2/replies",
+ "type": "OrderedCollection",
+ "totalItems": 0,
+ "first": "https://example.net/user/library_lurker/review/2/replies?page=1",
+ "last": "https://example.net/user/library_lurker/review/2/replies?page=1",
+ "@context": "https://www.w3.org/ns/activitystreams"
+ },
+ "summary": "Spoilers ahead!",
+ "tag": [],
+ "attachment": [],
+ "sensitive": true,
+ "inReplyToBook": "https://example.net/book/1",
+ "name": "What a cracking read",
+ "rating": 4.5,
+ "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Comment
+
+A `Comment` on a book mentions a book and has a message body, reading status, and progress indicator.
+
+Example:
+
+```json
+{
+ "id": "https://example.net/user/library_lurker/comment/9",
+ "type": "Comment",
+ "published": "2023-06-30T21:43:46.013132+00:00",
+ "attributedTo": "https://example.net/user/library_lurker",
+ "content": "This is a very enjoyable book so far.
",
+ "to": ["https://example.net/user/library_lurker/followers"],
+ "cc": [],
+ "replies": {
+ "id": "https://example.net/user/library_lurker/comment/9/replies",
+ "type": "OrderedCollection",
+ "totalItems": 0,
+ "first": "https://example.net/user/library_lurker/comment/9/replies?page=1",
+ "last": "https://example.net/user/library_lurker/comment/9/replies?page=1",
+ "@context": "https://www.w3.org/ns/activitystreams"
+ },
+ "summary": "Spoilers ahead!",
+ "tag": [],
+ "attachment": [],
+ "sensitive": true,
+ "inReplyToBook": "https://example.net/book/1",
+ "readingStatus": "reading",
+ "progress": 25,
+ "progressMode": "PG",
+ "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Quotation
+
+A quotation (aka "quote") has a message body, an excerpt from a book including position as a page number or percentage indicator, and mentions a book.
+
+Example:
+
+```json
+{
+ "id": "https://example.net/user/mouse/quotation/13",
+ "url": "https://example.net/user/mouse/quotation/13",
+ "inReplyTo": null,
+ "published": "2020-05-10T02:38:31.150343+00:00",
+ "attributedTo": "https://example.net/user/mouse",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://example.net/user/mouse/followers"
+ ],
+ "sensitive": false,
+ "content": "I really like this quote",
+ "type": "Quotation",
+ "replies": {
+ "id": "https://example.net/user/mouse/quotation/13/replies",
+ "type": "Collection",
+ "first": {
+ "type": "CollectionPage",
+ "next": "https://example.net/user/mouse/quotation/13/replies?only_other_accounts=true&page=true",
+ "partOf": "https://example.net/user/mouse/quotation/13/replies",
+ "items": []
+ }
+ },
+ "inReplyToBook": "https://example.net/book/1",
+ "quote": "To be or not to be, that is the question.",
+ "position": 50,
+ "positionMode": "PCT",
+ "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+### Custom Objects
+
+##### Work
+A particular book, a "work" in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
+
+Example:
+
+```json
+{
+ "id": "https://bookwyrm.social/book/5988",
+ "type": "Work",
+ "authors": [
+ "https://bookwyrm.social/author/417"
+ ],
+ "first_published_date": null,
+ "published_date": null,
+ "title": "Piranesi",
+ "sort_title": null,
+ "subtitle": null,
+ "description": "**From the *New York Times* bestselling author of *Jonathan Strange & Mr. Norrell*, an intoxicating, hypnotic new novel set in a dreamlike alternative reality.",
+ "languages": [],
+ "series": null,
+ "series_number": null,
+ "subjects": [
+ "English literature"
+ ],
+ "subject_places": [],
+ "openlibrary_key": "OL20893680W",
+ "librarything_key": null,
+ "goodreads_key": null,
+ "attachment": [
+ {
+ "url": "https://bookwyrm.social/images/covers/10226290-M.jpg",
+ "type": "Image"
+ }
+ ],
+ "lccn": null,
+ "editions": [
+ "https://bookwyrm.social/book/5989"
+ ],
+ "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Edition
+A particular _manifestation_ of a Work, in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
+
+Example:
+
+```json
+{
+ "id": "https://bookwyrm.social/book/5989",
+ "lastEditedBy": "https://example.net/users/rat",
+ "type": "Edition",
+ "authors": [
+ "https://bookwyrm.social/author/417"
+ ],
+ "first_published_date": null,
+ "published_date": "2020-09-15T00:00:00+00:00",
+ "title": "Piranesi",
+ "sort_title": null,
+ "subtitle": null,
+ "description": "Piranesi's house is no ordinary building; its rooms are infinite, its corridors endless, its walls are lined with thousands upon thousands of statues, each one different from all the others.",
+ "languages": [
+ "English"
+ ],
+ "series": null,
+ "series_number": null,
+ "subjects": [],
+ "subject_places": [],
+ "openlibrary_key": "OL29486417M",
+ "librarything_key": null,
+ "goodreads_key": null,
+ "isfdb": null,
+ "attachment": [
+ {
+ "url": "https://bookwyrm.social/images/covers/50202953._SX318_.jpg",
+ "type": "Image"
+ }
+ ],
+ "isbn_10": "1526622424",
+ "isbn_13": "9781526622426",
+ "oclc_number": null,
+ "asin": null,
+ "pages": 272,
+ "physical_format": null,
+ "publishers": [
+ "Bloomsbury Publishing Plc"
+ ],
+ "work": "https://bookwyrm.social/book/5988",
+ "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### Shelf
+
+A user's book collection. By default, every user has a `to-read`, `reading`, `read`, and `stopped-reading` shelf which are used to track reading progress. Users may create an unlimited number of additional shelves with their own ids.
+
+Example
+
+```json
+{
+ "id": "https://example.net/user/avid_reader/books/extraspecialbooks-5",
+ "type": "Shelf",
+ "totalItems": 0,
+ "first": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
+ "last": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
+ "name": "Extra special books",
+ "owner": "https://example.net/user/avid_reader",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://example.net/user/avid_reader/followers"
+ ],
+ "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### List
+
+A collection of books that may have items contributed by users other than the one who created the list.
+
+Example:
+
+```json
+{
+ "id": "https://example.net/list/1",
+ "type": "BookList",
+ "totalItems": 0,
+ "first": "https://example.net/list/1?page=1",
+ "last": "https://example.net/list/1?page=1",
+ "name": "My cool list",
+ "owner": "https://example.net/user/avid_reader",
+ "to": [
+ "https://www.w3.org/ns/activitystreams#Public"
+ ],
+ "cc": [
+ "https://example.net/user/avid_reader/followers"
+ ],
+ "summary": "A list of books I like.",
+ "curation": "curated",
+ "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### Activities
+
+- `Create`: Adds a shelf or list to the database.
+- `Delete`: Removes a shelf or list.
+- `Add`: Adds a book to a shelf or list.
+- `Remove`: Removes a book from a shelf or list.
+
+## Alternative Serialization
+Because BookWyrm uses custom object types that aren't listed in [the standard ActivityStreams Vocabulary](https://www.w3.org/TR/activitystreams-vocabulary), some statuses are transformed into standard types when sent to or viewed by non-BookWyrm services. `Review`s are converted into `Article`s, and `Comment`s and `Quotation`s are converted into `Note`s, with a link to the book and the cover image attached.
+
+In future this may be done with [JSON-LD type arrays](https://www.w3.org/TR/json-ld/#specifying-the-type) instead.
+
+## Other extensions
+
+### Webfinger
+
+Bookwyrm uses the [Webfinger](https://datatracker.ietf.org/doc/html/rfc7033) standard to identify and disambiguate fediverse actors. The [Webfinger documentation on the Mastodon project](https://docs.joinmastodon.org/spec/webfinger/) provides a good overview of how Webfinger is used.
+
+### HTTP Signatures
+
+Bookwyrm uses and requires HTTP signatures for all `POST` requests. `GET` requests are not signed by default, but if Bookwyrm receives a `403` response to a `GET` it will re-send the request, signed by the default server user. This usually will have a user id of `https://example.net/user/bookwyrm.instance.actor`
+
+#### publicKey id
+
+In older versions of Bookwyrm the `publicKey.id` was incorrectly listed in request headers as `https://example.net/user/username#main-key`. As of v0.6.3 the id is now listed correctly, as `https://example.net/user/username/#main-key`. In most ActivityPub implementations this will make no difference as the URL will usually resolve to the same place.
+
+### NodeInfo
+
+Bookwyrm uses the [NodeInfo](http://nodeinfo.diaspora.software/) standard to provide statistics and version information for each instance.
+
+## Further Documentation
+
+See [docs.joinbookwyrm.com/](https://docs.joinbookwyrm.com/) for more documentation.
\ No newline at end of file
diff --git a/README.md b/README.md
index 558d42d45..f8b2eb1f6 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ Keep track of what books you've read, and what books you'd like to read in the f
Federation allows you to interact with users on other instances and services, and also shares metadata about books and authors, which collaboratively builds a decentralized database of books.
### Privacy and moderation
-Users and administrators can control who can see thier posts and what other instances to federate with.
+Users and administrators can control who can see their posts and what other instances to federate with.
## Tech Stack
Web backend
diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py
index bfb22fa32..2697620f0 100644
--- a/bookwyrm/activitypub/__init__.py
+++ b/bookwyrm/activitypub/__init__.py
@@ -3,8 +3,12 @@ import inspect
import sys
from .base_activity import ActivityEncoder, Signature, naive_parse
-from .base_activity import Link, Mention
-from .base_activity import ActivitySerializerError, resolve_remote_id
+from .base_activity import Link, Mention, Hashtag
+from .base_activity import (
+ ActivitySerializerError,
+ resolve_remote_id,
+ get_representative,
+)
from .image import Document, Image
from .note import Note, GeneratedNote, Article, Comment, Quotation
from .note import Review, Rating
diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py
index e942c9aeb..05e7d8a05 100644
--- a/bookwyrm/activitypub/base_activity.py
+++ b/bookwyrm/activitypub/base_activity.py
@@ -1,16 +1,27 @@
""" basics for an activitypub serializer """
+from __future__ import annotations
from dataclasses import dataclass, fields, MISSING
from json import JSONEncoder
import logging
+from typing import Optional, Union, TypeVar, overload, Any
+
+import requests
from django.apps import apps
from django.db import IntegrityError, transaction
+from django.utils.http import http_date
+from bookwyrm import models
from bookwyrm.connectors import ConnectorException, get_data
-from bookwyrm.tasks import app, MEDIUM
+from bookwyrm.models import base_model
+from bookwyrm.signatures import make_signature
+from bookwyrm.settings import DOMAIN, INSTANCE_ACTOR_USERNAME
+from bookwyrm.tasks import app, MISC
logger = logging.getLogger(__name__)
+TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel)
+
class ActivitySerializerError(ValueError):
"""routine problems serializing activitypub json"""
@@ -60,7 +71,13 @@ class ActivityObject:
id: str
type: str
- def __init__(self, activity_objects=None, **kwargs):
+ def __init__(
+ self,
+ activity_objects: Optional[
+ dict[str, Union[str, list[str], ActivityObject, base_model.BookWyrmModel]]
+ ] = None,
+ **kwargs: Any,
+ ):
"""this lets you pass in an object with fields that aren't in the
dataclass, which it ignores. Any field in the dataclass is required or
has a default value"""
@@ -95,16 +112,34 @@ class ActivityObject:
# pylint: disable=too-many-locals,too-many-branches,too-many-arguments
def to_model(
- self, model=None, instance=None, allow_create=True, save=True, overwrite=True
- ):
- """convert from an activity to a model instance"""
+ self,
+ model: Optional[type[TBookWyrmModel]] = None,
+ instance: Optional[TBookWyrmModel] = None,
+ allow_create: bool = True,
+ save: bool = True,
+ overwrite: bool = True,
+ allow_external_connections: bool = True,
+ ) -> Optional[TBookWyrmModel]:
+ """convert from an activity to a model instance. Args:
+ model: the django model that this object is being converted to
+ (will guess if not known)
+ instance: an existing database entry that is going to be updated by
+ this activity
+ allow_create: whether a new object should be created if there is no
+ existing object is provided or found matching the remote_id
+ save: store in the database if true, return an unsaved model obj if false
+ overwrite: replace fields in the database with this activity if true,
+ only update blank fields if false
+ allow_external_connections: look up missing data if true,
+ throw an exception if false and an external connection is needed
+ """
model = model or get_model_from_type(self.type)
# only reject statuses if we're potentially creating them
if (
allow_create
and hasattr(model, "ignore_activity")
- and model.ignore_activity(self)
+ and model.ignore_activity(self, allow_external_connections)
):
return None
@@ -122,7 +157,10 @@ class ActivityObject:
for field in instance.simple_fields:
try:
changed = field.set_field_from_activity(
- instance, self, overwrite=overwrite
+ instance,
+ self,
+ overwrite=overwrite,
+ allow_external_connections=allow_external_connections,
)
if changed:
update_fields.append(field.name)
@@ -133,7 +171,11 @@ class ActivityObject:
# too early and jank up users
for field in instance.image_fields:
changed = field.set_field_from_activity(
- instance, self, save=save, overwrite=overwrite
+ instance,
+ self,
+ save=save,
+ overwrite=overwrite,
+ allow_external_connections=allow_external_connections,
)
if changed:
update_fields.append(field.name)
@@ -156,8 +198,12 @@ class ActivityObject:
# add many to many fields, which have to be set post-save
for field in instance.many_to_many_fields:
- # mention books/users, for example
- field.set_field_from_activity(instance, self)
+ # mention books/users/hashtags, for example
+ field.set_field_from_activity(
+ instance,
+ self,
+ allow_external_connections=allow_external_connections,
+ )
# reversed relationships in the models
for (
@@ -194,6 +240,11 @@ class ActivityObject:
try:
if issubclass(type(v), ActivityObject):
data[k] = v.serialize()
+ elif isinstance(v, list):
+ data[k] = [
+ e.serialize() if issubclass(type(e), ActivityObject) else e
+ for e in v
+ ]
except TypeError:
pass
data = {k: v for (k, v) in data.items() if v is not None and k not in omit}
@@ -202,7 +253,7 @@ class ActivityObject:
return data
-@app.task(queue=MEDIUM)
+@app.task(queue=MISC)
@transaction.atomic
def set_related_field(
model_name, origin_model_name, related_field_name, related_remote_id, data
@@ -241,10 +292,10 @@ def set_related_field(
def get_model_from_type(activity_type):
"""given the activity, what type of model"""
- models = apps.get_models()
+ activity_models = apps.get_models()
model = [
m
- for m in models
+ for m in activity_models
if hasattr(m, "activity_serializer")
and hasattr(m.activity_serializer, "type")
and m.activity_serializer.type == activity_type
@@ -256,10 +307,48 @@ def get_model_from_type(activity_type):
return model[0]
+# pylint: disable=too-many-arguments
+@overload
def resolve_remote_id(
- remote_id, model=None, refresh=False, save=True, get_activity=False
-):
- """take a remote_id and return an instance, creating if necessary"""
+ remote_id: str,
+ model: type[TBookWyrmModel],
+ refresh: bool = False,
+ save: bool = True,
+ get_activity: bool = False,
+ allow_external_connections: bool = True,
+) -> TBookWyrmModel:
+ ...
+
+
+# pylint: disable=too-many-arguments
+@overload
+def resolve_remote_id(
+ remote_id: str,
+ model: Optional[str] = None,
+ refresh: bool = False,
+ save: bool = True,
+ get_activity: bool = False,
+ allow_external_connections: bool = True,
+) -> base_model.BookWyrmModel:
+ ...
+
+
+# pylint: disable=too-many-arguments
+def resolve_remote_id(
+ remote_id: str,
+ model: Optional[Union[str, type[base_model.BookWyrmModel]]] = None,
+ refresh: bool = False,
+ save: bool = True,
+ get_activity: bool = False,
+ allow_external_connections: bool = True,
+) -> base_model.BookWyrmModel:
+ """take a remote_id and return an instance, creating if necessary. Args:
+ remote_id: the unique url for looking up the object in the db or by http
+ model: a string or object representing the model that corresponds to the object
+ save: whether to return an unsaved database entry or a saved one
+ get_activity: whether to return the activitypub object or the model object
+ allow_external_connections: whether to make http connections
+ """
if model: # a bonus check we can do if we already know the model
if isinstance(model, str):
model = apps.get_model(f"bookwyrm.{model}", require_ready=True)
@@ -267,13 +356,26 @@ def resolve_remote_id(
if result and not refresh:
return result if not get_activity else result.to_activity_dataclass()
+ # The above block will return the object if it already exists in the database.
+ # If it doesn't, an external connection would be needed, so check if that's cool
+ if not allow_external_connections:
+ raise ActivitySerializerError(
+ "Unable to serialize object without making external HTTP requests"
+ )
+
# load the data and create the object
try:
data = get_data(remote_id)
- except ConnectorException:
- logger.exception("Could not connect to host for remote_id: %s", remote_id)
+ except ConnectionError:
+ logger.info("Could not connect to host for remote_id: %s", remote_id)
return None
-
+ except requests.HTTPError as e:
+ if (e.response is not None) and e.response.status_code == 401:
+ # This most likely means it's a mastodon with secure fetch enabled.
+ data = get_activitypub_data(remote_id)
+ else:
+ logger.info("Could not connect to host for remote_id: %s", remote_id)
+ return None
# determine the model implicitly, if not provided
# or if it's a model with subclasses like Status, check again
if not model or hasattr(model.objects, "select_subclasses"):
@@ -292,6 +394,52 @@ def resolve_remote_id(
return item.to_model(model=model, instance=result, save=save)
+def get_representative():
+ """Get or create an actor representing the instance
+ to sign requests to 'secure mastodon' servers"""
+ username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
+ email = "bookwyrm@localhost"
+ try:
+ user = models.User.objects.get(username=username)
+ except models.User.DoesNotExist:
+ user = models.User.objects.create_user(
+ username=username,
+ email=email,
+ local=True,
+ localname=INSTANCE_ACTOR_USERNAME,
+ )
+ return user
+
+
+def get_activitypub_data(url):
+ """wrapper for request.get"""
+ now = http_date()
+ sender = get_representative()
+ if not sender.key_pair.private_key:
+ # this shouldn't happen. it would be bad if it happened.
+ raise ValueError("No private key found for sender")
+ try:
+ resp = requests.get(
+ url,
+ headers={
+ # pylint: disable=line-too-long
+ "Accept": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+ "Date": now,
+ "Signature": make_signature("get", sender, url, now),
+ },
+ )
+ except requests.RequestException:
+ raise ConnectorException()
+ if not resp.ok:
+ resp.raise_for_status()
+ try:
+ data = resp.json()
+ except ValueError:
+ raise ConnectorException()
+
+ return data
+
+
@dataclass(init=False)
class Link(ActivityObject):
"""for tagging a book in a status"""
@@ -306,7 +454,9 @@ class Link(ActivityObject):
def serialize(self, **kwargs):
"""remove fields"""
- omit = ("id", "type", "@context")
+ omit = ("id", "@context")
+ if self.type == "Link":
+ omit += ("type",)
return super().serialize(omit=omit)
@@ -315,3 +465,10 @@ class Mention(Link):
"""a subtype of Link for mentioning an actor"""
type: str = "Mention"
+
+
+@dataclass(init=False)
+class Hashtag(Link):
+ """a subtype of Link for mentioning a hashtag"""
+
+ type: str = "Hashtag"
diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py
index e6a01b359..5db0dc3ac 100644
--- a/bookwyrm/activitypub/book.py
+++ b/bookwyrm/activitypub/book.py
@@ -1,6 +1,6 @@
""" book and author data """
from dataclasses import dataclass, field
-from typing import List
+from typing import Optional
from .base_activity import ActivityObject
from .image import Document
@@ -11,17 +11,19 @@ from .image import Document
class BookData(ActivityObject):
"""shared fields for all book data and authors"""
- openlibraryKey: str = None
- inventaireId: str = None
- librarythingKey: str = None
- goodreadsKey: str = None
- bnfId: str = None
- viaf: str = None
- wikidata: str = None
- asin: str = None
- lastEditedBy: str = None
- links: List[str] = field(default_factory=lambda: [])
- fileLinks: List[str] = field(default_factory=lambda: [])
+ openlibraryKey: Optional[str] = None
+ inventaireId: Optional[str] = None
+ librarythingKey: Optional[str] = None
+ goodreadsKey: Optional[str] = None
+ bnfId: Optional[str] = None
+ viaf: Optional[str] = None
+ wikidata: Optional[str] = None
+ asin: Optional[str] = None
+ aasin: Optional[str] = None
+ isfdb: Optional[str] = None
+ lastEditedBy: Optional[str] = None
+ links: list[str] = field(default_factory=list)
+ fileLinks: list[str] = field(default_factory=list)
# pylint: disable=invalid-name
@@ -33,17 +35,17 @@ class Book(BookData):
sortTitle: str = None
subtitle: str = None
description: str = ""
- languages: List[str] = field(default_factory=lambda: [])
+ languages: list[str] = field(default_factory=list)
series: str = ""
seriesNumber: str = ""
- subjects: List[str] = field(default_factory=lambda: [])
- subjectPlaces: List[str] = field(default_factory=lambda: [])
+ subjects: list[str] = field(default_factory=list)
+ subjectPlaces: list[str] = field(default_factory=list)
- authors: List[str] = field(default_factory=lambda: [])
+ authors: list[str] = field(default_factory=list)
firstPublishedDate: str = ""
publishedDate: str = ""
- cover: Document = None
+ cover: Optional[Document] = None
type: str = "Book"
@@ -56,10 +58,10 @@ class Edition(Book):
isbn10: str = ""
isbn13: str = ""
oclcNumber: str = ""
- pages: int = None
+ pages: Optional[int] = None
physicalFormat: str = ""
physicalFormatDetail: str = ""
- publishers: List[str] = field(default_factory=lambda: [])
+ publishers: list[str] = field(default_factory=list)
editionRank: int = 0
type: str = "Edition"
@@ -71,7 +73,7 @@ class Work(Book):
"""work instance of a book object"""
lccn: str = ""
- editions: List[str] = field(default_factory=lambda: [])
+ editions: list[str] = field(default_factory=list)
type: str = "Work"
@@ -81,12 +83,13 @@ class Author(BookData):
"""author of a book"""
name: str
- isni: str = None
- viafId: str = None
- gutenbergId: str = None
- born: str = None
- died: str = None
- aliases: List[str] = field(default_factory=lambda: [])
+ isni: Optional[str] = None
+ viafId: Optional[str] = None
+ gutenbergId: Optional[str] = None
+ born: Optional[str] = None
+ died: Optional[str] = None
+ aliases: list[str] = field(default_factory=list)
bio: str = ""
wikipediaLink: str = ""
type: str = "Author"
+ website: str = ""
diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py
index eb18b8b8a..6a081058c 100644
--- a/bookwyrm/activitypub/note.py
+++ b/bookwyrm/activitypub/note.py
@@ -1,9 +1,12 @@
""" note serializer and children thereof """
from dataclasses import dataclass, field
from typing import Dict, List
-from django.apps import apps
+import re
-from .base_activity import ActivityObject, Link
+from django.apps import apps
+from django.db import IntegrityError, transaction
+
+from .base_activity import ActivityObject, ActivitySerializerError, Link
from .image import Document
@@ -38,6 +41,47 @@ class Note(ActivityObject):
updated: str = None
type: str = "Note"
+ # pylint: disable=too-many-arguments
+ def to_model(
+ self,
+ model=None,
+ instance=None,
+ allow_create=True,
+ save=True,
+ overwrite=True,
+ allow_external_connections=True,
+ ):
+ instance = super().to_model(
+ model, instance, allow_create, save, overwrite, allow_external_connections
+ )
+
+ if instance is None:
+ return instance
+
+ # Replace links to hashtags in content with local URLs
+ changed_content = False
+ for hashtag in instance.mention_hashtags.all():
+ updated_content = re.sub(
+ rf'({hashtag.name})',
+ rf"\1{hashtag.remote_id}\2",
+ instance.content,
+ flags=re.IGNORECASE,
+ )
+ if instance.content != updated_content:
+ instance.content = updated_content
+ changed_content = True
+
+ if not save or not changed_content:
+ return instance
+
+ with transaction.atomic():
+ try:
+ instance.save(broadcast=False, update_fields=["content"])
+ except IntegrityError as e:
+ raise ActivitySerializerError(e)
+
+ return instance
+
@dataclass(init=False)
class Article(Note):
diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py
index 36898bc7e..4b7514b5a 100644
--- a/bookwyrm/activitypub/verbs.py
+++ b/bookwyrm/activitypub/verbs.py
@@ -14,12 +14,12 @@ class Verb(ActivityObject):
actor: str
object: ActivityObject
- def action(self):
+ def action(self, allow_external_connections=True):
"""usually we just want to update and save"""
# self.object may return None if the object is invalid in an expected way
# ie, Question type
if self.object:
- self.object.to_model()
+ self.object.to_model(allow_external_connections=allow_external_connections)
# pylint: disable=invalid-name
@@ -42,7 +42,7 @@ class Delete(Verb):
cc: List[str] = field(default_factory=lambda: [])
type: str = "Delete"
- def action(self):
+ def action(self, allow_external_connections=True):
"""find and delete the activity object"""
if not self.object:
return
@@ -52,7 +52,11 @@ class Delete(Verb):
model = apps.get_model("bookwyrm.User")
obj = model.find_existing_by_remote_id(self.object)
else:
- obj = self.object.to_model(save=False, allow_create=False)
+ obj = self.object.to_model(
+ save=False,
+ allow_create=False,
+ allow_external_connections=allow_external_connections,
+ )
if obj:
obj.delete()
@@ -67,11 +71,13 @@ class Update(Verb):
to: List[str]
type: str = "Update"
- def action(self):
+ def action(self, allow_external_connections=True):
"""update a model instance from the dataclass"""
if not self.object:
return
- self.object.to_model(allow_create=False)
+ self.object.to_model(
+ allow_create=False, allow_external_connections=allow_external_connections
+ )
@dataclass(init=False)
@@ -80,10 +86,10 @@ class Undo(Verb):
type: str = "Undo"
- def action(self):
+ def action(self, allow_external_connections=True):
"""find and remove the activity object"""
if isinstance(self.object, str):
- # it may be that sometihng should be done with these, but idk what
+ # it may be that something should be done with these, but idk what
# this seems just to be coming from pleroma
return
@@ -92,13 +98,28 @@ class Undo(Verb):
model = None
if self.object.type == "Follow":
model = apps.get_model("bookwyrm.UserFollows")
- obj = self.object.to_model(model=model, save=False, allow_create=False)
+ obj = self.object.to_model(
+ model=model,
+ save=False,
+ allow_create=False,
+ allow_external_connections=allow_external_connections,
+ )
if not obj:
- # this could be a folloq request not a follow proper
+ # this could be a follow request not a follow proper
model = apps.get_model("bookwyrm.UserFollowRequest")
- obj = self.object.to_model(model=model, save=False, allow_create=False)
+ obj = self.object.to_model(
+ model=model,
+ save=False,
+ allow_create=False,
+ allow_external_connections=allow_external_connections,
+ )
else:
- obj = self.object.to_model(model=model, save=False, allow_create=False)
+ obj = self.object.to_model(
+ model=model,
+ save=False,
+ allow_create=False,
+ allow_external_connections=allow_external_connections,
+ )
if not obj:
# if we don't have the object, we can't undo it. happens a lot with boosts
return
@@ -112,9 +133,9 @@ class Follow(Verb):
object: str
type: str = "Follow"
- def action(self):
+ def action(self, allow_external_connections=True):
"""relationship save"""
- self.to_model()
+ self.to_model(allow_external_connections=allow_external_connections)
@dataclass(init=False)
@@ -124,9 +145,9 @@ class Block(Verb):
object: str
type: str = "Block"
- def action(self):
+ def action(self, allow_external_connections=True):
"""relationship save"""
- self.to_model()
+ self.to_model(allow_external_connections=allow_external_connections)
@dataclass(init=False)
@@ -136,7 +157,7 @@ class Accept(Verb):
object: Follow
type: str = "Accept"
- def action(self):
+ def action(self, allow_external_connections=True):
"""accept a request"""
obj = self.object.to_model(save=False, allow_create=True)
obj.accept()
@@ -149,7 +170,7 @@ class Reject(Verb):
object: Follow
type: str = "Reject"
- def action(self):
+ def action(self, allow_external_connections=True):
"""reject a follow request"""
obj = self.object.to_model(save=False, allow_create=False)
obj.reject()
@@ -163,7 +184,7 @@ class Add(Verb):
object: CollectionItem
type: str = "Add"
- def action(self):
+ def action(self, allow_external_connections=True):
"""figure out the target to assign the item to a collection"""
target = resolve_remote_id(self.target)
item = self.object.to_model(save=False)
@@ -177,7 +198,7 @@ class Remove(Add):
type: str = "Remove"
- def action(self):
+ def action(self, allow_external_connections=True):
"""find and remove the activity object"""
obj = self.object.to_model(save=False, allow_create=False)
if obj:
@@ -191,9 +212,9 @@ class Like(Verb):
object: str
type: str = "Like"
- def action(self):
+ def action(self, allow_external_connections=True):
"""like"""
- self.to_model()
+ self.to_model(allow_external_connections=allow_external_connections)
# pylint: disable=invalid-name
@@ -207,6 +228,6 @@ class Announce(Verb):
object: str
type: str = "Announce"
- def action(self):
+ def action(self, allow_external_connections=True):
"""boost"""
- self.to_model()
+ self.to_model(allow_external_connections=allow_external_connections)
diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py
index f8312f063..42f99e209 100644
--- a/bookwyrm/activitystreams.py
+++ b/bookwyrm/activitystreams.py
@@ -4,27 +4,32 @@ from django.dispatch import receiver
from django.db import transaction
from django.db.models import signals, Q
from django.utils import timezone
+from opentelemetry import trace
from bookwyrm import models
from bookwyrm.redis_store import RedisStore, r
-from bookwyrm.tasks import app, LOW, MEDIUM, HIGH
+from bookwyrm.tasks import app, STREAMS, IMPORT_TRIGGERED
+from bookwyrm.telemetry import open_telemetry
+
+
+tracer = open_telemetry.tracer()
class ActivityStream(RedisStore):
"""a category of activity stream (like home, local, books)"""
- def stream_id(self, user):
+ def stream_id(self, user_id):
"""the redis key for this user's instance of this stream"""
- return f"{user.id}-{self.key}"
+ return f"{user_id}-{self.key}"
- def unread_id(self, user):
+ def unread_id(self, user_id):
"""the redis key for this user's unread count for this stream"""
- stream_id = self.stream_id(user)
+ stream_id = self.stream_id(user_id)
return f"{stream_id}-unread"
- def unread_by_status_type_id(self, user):
+ def unread_by_status_type_id(self, user_id):
"""the redis key for this user's unread count for this stream"""
- stream_id = self.stream_id(user)
+ stream_id = self.stream_id(user_id)
return f"{stream_id}-unread-by-type"
def get_rank(self, obj): # pylint: disable=no-self-use
@@ -33,16 +38,19 @@ class ActivityStream(RedisStore):
def add_status(self, status, increment_unread=False):
"""add a status to users' feeds"""
+ audience = self.get_audience(status)
# the pipeline contains all the add-to-stream activities
- pipeline = self.add_object_to_related_stores(status, execute=False)
+ pipeline = self.add_object_to_stores(
+ status, self.get_stores_for_users(audience), execute=False
+ )
if increment_unread:
- for user in self.get_audience(status):
+ for user_id in audience:
# add to the unread status count
- pipeline.incr(self.unread_id(user))
+ pipeline.incr(self.unread_id(user_id))
# add to the unread status count for status type
pipeline.hincrby(
- self.unread_by_status_type_id(user), get_status_type(status), 1
+ self.unread_by_status_type_id(user_id), get_status_type(status), 1
)
# and go!
@@ -52,21 +60,21 @@ class ActivityStream(RedisStore):
"""add a user's statuses to another user's feed"""
# only add the statuses that the viewer should be able to see (ie, not dms)
statuses = models.Status.privacy_filter(viewer).filter(user=user)
- self.bulk_add_objects_to_store(statuses, self.stream_id(viewer))
+ self.bulk_add_objects_to_store(statuses, self.stream_id(viewer.id))
def remove_user_statuses(self, viewer, user):
"""remove a user's status from another user's feed"""
# remove all so that followers only statuses are removed
statuses = user.status_set.all()
- self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer))
+ self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer.id))
def get_activity_stream(self, user):
"""load the statuses to be displayed"""
# clear unreads for this feed
- r.set(self.unread_id(user), 0)
- r.delete(self.unread_by_status_type_id(user))
+ r.set(self.unread_id(user.id), 0)
+ r.delete(self.unread_by_status_type_id(user.id))
- statuses = self.get_store(self.stream_id(user))
+ statuses = self.get_store(self.stream_id(user.id))
return (
models.Status.objects.select_subclasses()
.filter(id__in=statuses)
@@ -83,11 +91,11 @@ class ActivityStream(RedisStore):
def get_unread_count(self, user):
"""get the unread status count for this user's feed"""
- return int(r.get(self.unread_id(user)) or 0)
+ return int(r.get(self.unread_id(user.id)) or 0)
def get_unread_count_by_status_type(self, user):
"""get the unread status count for this user's feed's status types"""
- status_types = r.hgetall(self.unread_by_status_type_id(user))
+ status_types = r.hgetall(self.unread_by_status_type_id(user.id))
return {
str(key.decode("utf-8")): int(value) or 0
for key, value in status_types.items()
@@ -95,13 +103,20 @@ class ActivityStream(RedisStore):
def populate_streams(self, user):
"""go from zero to a timeline"""
- self.populate_store(self.stream_id(user))
+ self.populate_store(self.stream_id(user.id))
- def get_audience(self, status): # pylint: disable=no-self-use
- """given a status, what users should see it"""
- # direct messages don't appeard in feeds, direct comments/reviews/etc do
+ @tracer.start_as_current_span("ActivityStream._get_audience")
+ def _get_audience(self, status): # pylint: disable=no-self-use
+ """given a status, what users should see it, excluding the author"""
+ trace.get_current_span().set_attribute("status_type", status.status_type)
+ trace.get_current_span().set_attribute("status_privacy", status.privacy)
+ trace.get_current_span().set_attribute(
+ "status_reply_parent_privacy",
+ status.reply_parent.privacy if status.reply_parent else status.privacy,
+ )
+ # direct messages don't appear in feeds, direct comments/reviews/etc do
if status.privacy == "direct" and status.status_type == "Note":
- return []
+ return models.User.objects.none()
# everybody who could plausibly see this status
audience = models.User.objects.filter(
@@ -114,15 +129,13 @@ class ActivityStream(RedisStore):
# only visible to the poster and mentioned users
if status.privacy == "direct":
audience = audience.filter(
- Q(id=status.user.id) # if the user is the post's author
- | Q(id__in=status.mention_users.all()) # if the user is mentioned
+ Q(id__in=status.mention_users.all()) # if the user is mentioned
)
# don't show replies to statuses the user can't see
elif status.reply_parent and status.reply_parent.privacy == "followers":
audience = audience.filter(
- Q(id=status.user.id) # if the user is the post's author
- | Q(id=status.reply_parent.user.id) # if the user is the OG author
+ Q(id=status.reply_parent.user.id) # if the user is the OG author
| (
Q(following=status.user) & Q(following=status.reply_parent.user)
) # if the user is following both authors
@@ -131,13 +144,23 @@ class ActivityStream(RedisStore):
# only visible to the poster's followers and tagged users
elif status.privacy == "followers":
audience = audience.filter(
- Q(id=status.user.id) # if the user is the post's author
- | Q(following=status.user) # if the user is following the author
+ Q(following=status.user) # if the user is following the author
)
return audience.distinct()
- def get_stores_for_object(self, obj):
- return [self.stream_id(u) for u in self.get_audience(obj)]
+ @tracer.start_as_current_span("ActivityStream.get_audience")
+ def get_audience(self, status):
+ """given a status, what users should see it"""
+ trace.get_current_span().set_attribute("stream_id", self.key)
+ audience = self._get_audience(status).values_list("id", flat=True)
+ status_author = models.User.objects.filter(
+ is_active=True, local=True, id=status.user.id
+ ).values_list("id", flat=True)
+ return list(set(list(audience) + list(status_author)))
+
+ def get_stores_for_users(self, user_ids):
+ """convert a list of user ids into redis store ids"""
+ return [self.stream_id(user_id) for user_id in user_ids]
def get_statuses_for_user(self, user): # pylint: disable=no-self-use
"""given a user, what statuses should they see on this stream"""
@@ -156,14 +179,19 @@ class HomeStream(ActivityStream):
key = "home"
+ @tracer.start_as_current_span("HomeStream.get_audience")
def get_audience(self, status):
- audience = super().get_audience(status)
+ trace.get_current_span().set_attribute("stream_id", self.key)
+ audience = super()._get_audience(status)
if not audience:
return []
- return audience.filter(
- Q(id=status.user.id) # if the user is the post's author
- | Q(following=status.user) # if the user is following the author
- ).distinct()
+ # if the user is following the author
+ audience = audience.filter(following=status.user).values_list("id", flat=True)
+ # if the user is the post's author
+ status_author = models.User.objects.filter(
+ is_active=True, local=True, id=status.user.id
+ ).values_list("id", flat=True)
+ return list(set(list(audience) + list(status_author)))
def get_statuses_for_user(self, user):
return models.Status.privacy_filter(
@@ -202,8 +230,20 @@ class BooksStream(ActivityStream):
key = "books"
- def get_audience(self, status):
+ def _get_audience(self, status):
"""anyone with the mentioned book on their shelves"""
+ work = (
+ status.book.parent_work
+ if hasattr(status, "book")
+ else status.mention_books.first().parent_work
+ )
+
+ audience = super()._get_audience(status)
+ if not audience:
+ return models.User.objects.none()
+ return audience.filter(shelfbook__book__parent_work=work).distinct()
+
+ def get_audience(self, status):
# only show public statuses on the books feed,
# and only statuses that mention books
if status.privacy != "public" or not (
@@ -211,16 +251,7 @@ class BooksStream(ActivityStream):
):
return []
- work = (
- status.book.parent_work
- if hasattr(status, "book")
- else status.mention_books.first().parent_work
- )
-
- audience = super().get_audience(status)
- if not audience:
- return []
- return audience.filter(shelfbook__book__parent_work=work).distinct()
+ return super().get_audience(status)
def get_statuses_for_user(self, user):
"""any public status that mentions the user's books"""
@@ -244,38 +275,38 @@ class BooksStream(ActivityStream):
def add_book_statuses(self, user, book):
"""add statuses about a book to a user's feed"""
work = book.parent_work
- statuses = (
- models.Status.privacy_filter(
- user,
- privacy_levels=["public"],
- )
- .filter(
- Q(comment__book__parent_work=work)
- | Q(quotation__book__parent_work=work)
- | Q(review__book__parent_work=work)
- | Q(mention_books__parent_work=work)
- )
- .distinct()
+ statuses = models.Status.privacy_filter(
+ user,
+ privacy_levels=["public"],
)
- self.bulk_add_objects_to_store(statuses, self.stream_id(user))
+
+ book_comments = statuses.filter(Q(comment__book__parent_work=work))
+ book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
+ book_reviews = statuses.filter(Q(review__book__parent_work=work))
+ book_mentions = statuses.filter(Q(mention_books__parent_work=work))
+
+ self.bulk_add_objects_to_store(book_comments, self.stream_id(user.id))
+ self.bulk_add_objects_to_store(book_quotations, self.stream_id(user.id))
+ self.bulk_add_objects_to_store(book_reviews, self.stream_id(user.id))
+ self.bulk_add_objects_to_store(book_mentions, self.stream_id(user.id))
def remove_book_statuses(self, user, book):
"""add statuses about a book to a user's feed"""
work = book.parent_work
- statuses = (
- models.Status.privacy_filter(
- user,
- privacy_levels=["public"],
- )
- .filter(
- Q(comment__book__parent_work=work)
- | Q(quotation__book__parent_work=work)
- | Q(review__book__parent_work=work)
- | Q(mention_books__parent_work=work)
- )
- .distinct()
+ statuses = models.Status.privacy_filter(
+ user,
+ privacy_levels=["public"],
)
- self.bulk_remove_objects_from_store(statuses, self.stream_id(user))
+
+ book_comments = statuses.filter(Q(comment__book__parent_work=work))
+ book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
+ book_reviews = statuses.filter(Q(review__book__parent_work=work))
+ book_mentions = statuses.filter(Q(mention_books__parent_work=work))
+
+ self.bulk_remove_objects_from_store(book_comments, self.stream_id(user.id))
+ self.bulk_remove_objects_from_store(book_quotations, self.stream_id(user.id))
+ self.bulk_remove_objects_from_store(book_reviews, self.stream_id(user.id))
+ self.bulk_remove_objects_from_store(book_mentions, self.stream_id(user.id))
# determine which streams are enabled in settings.py
@@ -298,6 +329,11 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
remove_status_task.delay(instance.id)
return
+ # We don't want to create multiple add_status_tasks for each status, and because
+ # the transactions are atomic, on_commit won't run until the status is ready to add.
+ if not created:
+ return
+
# when creating new things, gotta wait on the transaction
transaction.on_commit(
lambda: add_status_on_create_command(sender, instance, created)
@@ -306,13 +342,21 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
def add_status_on_create_command(sender, instance, created):
"""runs this code only after the database commit completes"""
- priority = HIGH
+ # boosts trigger 'saves" twice, so don't bother duplicating the task
+ if sender == models.Boost and not created:
+ return
+
+ priority = STREAMS
# check if this is an old status, de-prioritize if so
# (this will happen if federation is very slow, or, more expectedly, on csv import)
if instance.published_date < timezone.now() - timedelta(
days=1
) or instance.created_date < instance.published_date - timedelta(days=1):
- priority = LOW
+ # a backdated status from a local user is an import, don't add it
+ if instance.user.local:
+ return
+ # an out of date remote status is a low priority but should be added
+ priority = IMPORT_TRIGGERED
add_status_task.apply_async(
args=(instance.id,),
@@ -456,7 +500,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs):
# ---- TASKS
-@app.task(queue=LOW)
+@app.task(queue=STREAMS)
def add_book_statuses_task(user_id, book_id):
"""add statuses related to a book on shelve"""
user = models.User.objects.get(id=user_id)
@@ -464,7 +508,7 @@ def add_book_statuses_task(user_id, book_id):
BooksStream().add_book_statuses(user, book)
-@app.task(queue=LOW)
+@app.task(queue=STREAMS)
def remove_book_statuses_task(user_id, book_id):
"""remove statuses about a book from a user's books feed"""
user = models.User.objects.get(id=user_id)
@@ -472,7 +516,7 @@ def remove_book_statuses_task(user_id, book_id):
BooksStream().remove_book_statuses(user, book)
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
def populate_stream_task(stream, user_id):
"""background task for populating an empty activitystream"""
user = models.User.objects.get(id=user_id)
@@ -480,7 +524,7 @@ def populate_stream_task(stream, user_id):
stream.populate_streams(user)
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
def remove_status_task(status_ids):
"""remove a status from any stream it might be in"""
# this can take an id or a list of ids
@@ -490,10 +534,12 @@ def remove_status_task(status_ids):
for stream in streams.values():
for status in statuses:
- stream.remove_object_from_related_stores(status)
+ stream.remove_object_from_stores(
+ status, stream.get_stores_for_users(stream.get_audience(status))
+ )
-@app.task(queue=HIGH)
+@app.task(queue=STREAMS)
def add_status_task(status_id, increment_unread=False):
"""add a status to any stream it should be in"""
status = models.Status.objects.select_subclasses().get(id=status_id)
@@ -505,7 +551,7 @@ def add_status_task(status_id, increment_unread=False):
stream.add_status(status, increment_unread=increment_unread)
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
"""remove all statuses by a user from a viewer's stream"""
stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -515,7 +561,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
stream.remove_user_statuses(viewer, user)
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
def add_user_statuses_task(viewer_id, user_id, stream_list=None):
"""add all statuses by a user to a viewer's stream"""
stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -525,7 +571,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None):
stream.add_user_statuses(viewer, user)
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
def handle_boost_task(boost_id):
"""remove the original post and other, earlier boosts"""
instance = models.Status.objects.get(id=boost_id)
@@ -539,10 +585,10 @@ def handle_boost_task(boost_id):
for stream in streams.values():
# people who should see the boost (not people who see the original status)
- audience = stream.get_stores_for_object(instance)
- stream.remove_object_from_related_stores(boosted, stores=audience)
+ audience = stream.get_stores_for_users(stream.get_audience(instance))
+ stream.remove_object_from_stores(boosted, audience)
for status in old_versions:
- stream.remove_object_from_related_stores(status, stores=audience)
+ stream.remove_object_from_stores(status, audience)
def get_status_type(status):
diff --git a/bookwyrm/apps.py b/bookwyrm/apps.py
index 786f86e1c..b0c3e3fa4 100644
--- a/bookwyrm/apps.py
+++ b/bookwyrm/apps.py
@@ -35,11 +35,12 @@ class BookwyrmConfig(AppConfig):
# pylint: disable=no-self-use
def ready(self):
"""set up OTLP and preview image files, if desired"""
- if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+ if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE:
# pylint: disable=import-outside-toplevel
from bookwyrm.telemetry import open_telemetry
open_telemetry.instrumentDjango()
+ open_telemetry.instrumentPostgres()
if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS:
# Download any fonts that we don't have yet
diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py
index 803641cb7..ceb228f40 100644
--- a/bookwyrm/book_search.py
+++ b/bookwyrm/book_search.py
@@ -1,24 +1,62 @@
""" using a bookwyrm instance as a source of book data """
+from __future__ import annotations
from dataclasses import asdict, dataclass
from functools import reduce
import operator
+from typing import Optional, Union, Any, Literal, overload
from django.contrib.postgres.search import SearchRank, SearchQuery
-from django.db.models import OuterRef, Subquery, F, Q
+from django.db.models import F, Q
+from django.db.models.query import QuerySet
from bookwyrm import models
from bookwyrm import connectors
from bookwyrm.settings import MEDIA_FULL_URL
+@overload
+def search(
+ query: str,
+ *,
+ min_confidence: float = 0,
+ filters: Optional[list[Any]] = None,
+ return_first: Literal[False],
+) -> QuerySet[models.Edition]:
+ ...
+
+
+@overload
+def search(
+ query: str,
+ *,
+ min_confidence: float = 0,
+ filters: Optional[list[Any]] = None,
+ return_first: Literal[True],
+) -> Optional[models.Edition]:
+ ...
+
+
# pylint: disable=arguments-differ
-def search(query, min_confidence=0, filters=None, return_first=False):
+def search(
+ query: str,
+ *,
+ min_confidence: float = 0,
+ filters: Optional[list[Any]] = None,
+ return_first: bool = False,
+) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
"""search your local database"""
filters = filters or []
if not query:
- return []
- # first, try searching unqiue identifiers
- results = search_identifiers(query, *filters, return_first=return_first)
+ return None if return_first else []
+ query = query.strip()
+
+ results = None
+ # first, try searching unique identifiers
+ # unique identifiers never have spaces, title/author usually do
+ if not " " in query:
+ results = search_identifiers(query, *filters, return_first=return_first)
+
+ # if there were no identifier results...
if not results:
# then try searching title/author
results = search_title_author(
@@ -35,24 +73,10 @@ def isbn_search(query):
# If the ISBN has only 9 characters, prepend missing zero
query = query.strip().upper().rjust(10, "0")
filters = [{f: query} for f in ["isbn_10", "isbn_13"]]
- results = models.Edition.objects.filter(
+ return models.Edition.objects.filter(
reduce(operator.or_, (Q(**f) for f in filters))
).distinct()
- # when there are multiple editions of the same work, pick the default.
- # it would be odd for this to happen.
-
- default_editions = models.Edition.objects.filter(
- parent_work=OuterRef("parent_work")
- ).order_by("-edition_rank")
- results = (
- results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
- default_id=F("id")
- )
- or results
- )
- return results
-
def format_search_result(search_result):
"""convert a book object into a search result object"""
@@ -73,7 +97,9 @@ def format_search_result(search_result):
).json()
-def search_identifiers(query, *filters, return_first=False):
+def search_identifiers(
+ query, *filters, return_first=False
+) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
"""tries remote_id, isbn; defined as dedupe fields on the model"""
if connectors.maybe_isbn(query):
# Oh did you think the 'S' in ISBN stood for 'standard'?
@@ -88,28 +114,15 @@ def search_identifiers(query, *filters, return_first=False):
results = models.Edition.objects.filter(
*filters, reduce(operator.or_, (Q(**f) for f in or_filters))
).distinct()
- if results.count() <= 1:
- if return_first:
- return results.first()
- return results
- # when there are multiple editions of the same work, pick the default.
- # it would be odd for this to happen.
- default_editions = models.Edition.objects.filter(
- parent_work=OuterRef("parent_work")
- ).order_by("-edition_rank")
- results = (
- results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
- default_id=F("id")
- )
- or results
- )
if return_first:
return results.first()
return results
-def search_title_author(query, min_confidence, *filters, return_first=False):
+def search_title_author(
+ query, min_confidence, *filters, return_first=False
+) -> QuerySet[models.Edition]:
"""searches for title and author"""
query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
results = (
@@ -120,19 +133,16 @@ def search_title_author(query, min_confidence, *filters, return_first=False):
)
# when there are multiple editions of the same work, pick the closest
- editions_of_work = results.values("parent_work__id").values_list("parent_work__id")
+ editions_of_work = results.values_list("parent_work__id", flat=True).distinct()
# filter out multiple editions of the same work
list_results = []
- for work_id in set(editions_of_work):
- editions = results.filter(parent_work=work_id)
- default = editions.order_by("-edition_rank").first()
- default_rank = default.rank if default else 0
- # if mutliple books have the top rank, pick the default edition
- if default_rank == editions.first().rank:
- result = default
- else:
- result = editions.first()
+ for work_id in set(editions_of_work[:30]):
+ result = (
+ results.filter(parent_work=work_id)
+ .order_by("-rank", "-edition_rank")
+ .first()
+ )
if return_first:
return result
@@ -147,11 +157,11 @@ class SearchResult:
title: str
key: str
connector: object
- view_link: str = None
- author: str = None
- year: str = None
- cover: str = None
- confidence: int = 1
+ view_link: Optional[str] = None
+ author: Optional[str] = None
+ year: Optional[str] = None
+ cover: Optional[str] = None
+ confidence: float = 1.0
def __repr__(self):
# pylint: disable=consider-using-f-string
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index c1ee7fe78..8b6dcb885 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -1,44 +1,55 @@
""" functionality outline for a book data connector """
+from __future__ import annotations
from abc import ABC, abstractmethod
+from typing import Optional, TypedDict, Any, Callable, Union, Iterator
+from urllib.parse import quote_plus
import imghdr
import logging
import re
+import asyncio
+import requests
+from requests.exceptions import RequestException
+import aiohttp
from django.core.files.base import ContentFile
from django.db import transaction
-import requests
-from requests.exceptions import RequestException
from bookwyrm import activitypub, models, settings
+from bookwyrm.settings import USER_AGENT
from .connector_manager import load_more_data, ConnectorException, raise_not_valid_url
from .format_mappings import format_mappings
-
+from ..book_search import SearchResult
logger = logging.getLogger(__name__)
+JsonDict = dict[str, Any]
+
+
+class ConnectorResults(TypedDict):
+ """TypedDict for results returned by connector"""
+
+ connector: AbstractMinimalConnector
+ results: list[SearchResult]
+
class AbstractMinimalConnector(ABC):
"""just the bare bones, for other bookwyrm instances"""
- def __init__(self, identifier):
+ def __init__(self, identifier: str):
# load connector settings
info = models.Connector.objects.get(identifier=identifier)
self.connector = info
# the things in the connector model to copy over
- self_fields = [
- "base_url",
- "books_url",
- "covers_url",
- "search_url",
- "isbn_search_url",
- "name",
- "identifier",
- ]
- for field in self_fields:
- setattr(self, field, getattr(info, field))
+ self.base_url = info.base_url
+ self.books_url = info.books_url
+ self.covers_url = info.covers_url
+ self.search_url = info.search_url
+ self.isbn_search_url = info.isbn_search_url
+ self.name = info.name
+ self.identifier = info.identifier
- def get_search_url(self, query):
+ def get_search_url(self, query: str) -> str:
"""format the query url"""
# Check if the query resembles an ISBN
if maybe_isbn(query) and self.isbn_search_url and self.isbn_search_url != "":
@@ -48,44 +59,93 @@ class AbstractMinimalConnector(ABC):
return f"{self.isbn_search_url}{normalized_query}"
# NOTE: previously, we tried searching isbn and if that produces no results,
# searched as free text. This, instead, only searches isbn if it's isbn-y
- return f"{self.search_url}{query}"
+ return f"{self.search_url}{quote_plus(query)}"
- def process_search_response(self, query, data, min_confidence):
- """Format the search results based on the formt of the query"""
+ def process_search_response(
+ self, query: str, data: Any, min_confidence: float
+ ) -> list[SearchResult]:
+ """Format the search results based on the format of the query"""
if maybe_isbn(query):
return list(self.parse_isbn_search_data(data))[:10]
return list(self.parse_search_data(data, min_confidence))[:10]
+ async def get_results(
+ self,
+ session: aiohttp.ClientSession,
+ url: str,
+ min_confidence: float,
+ query: str,
+ ) -> Optional[ConnectorResults]:
+ """try this specific connector"""
+ # pylint: disable=line-too-long
+ headers = {
+ "Accept": (
+ 'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
+ ),
+ "User-Agent": USER_AGENT,
+ }
+ params = {"min_confidence": min_confidence}
+ try:
+ async with session.get(url, headers=headers, params=params) as response:
+ if not response.ok:
+ logger.info("Unable to connect to %s: %s", url, response.reason)
+ return None
+
+ try:
+ raw_data = await response.json()
+ except aiohttp.client_exceptions.ContentTypeError as err:
+ logger.exception(err)
+ return None
+
+ return ConnectorResults(
+ connector=self,
+ results=self.process_search_response(
+ query, raw_data, min_confidence
+ ),
+ )
+ except asyncio.TimeoutError:
+ logger.info("Connection timed out for url: %s", url)
+ except aiohttp.ClientError as err:
+ logger.info(err)
+ return None
+
@abstractmethod
- def get_or_create_book(self, remote_id):
+ def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
"""pull up a book record by whatever means possible"""
@abstractmethod
- def parse_search_data(self, data, min_confidence):
+ def parse_search_data(
+ self, data: Any, min_confidence: float
+ ) -> Iterator[SearchResult]:
"""turn the result json from a search into a list"""
@abstractmethod
- def parse_isbn_search_data(self, data):
+ def parse_isbn_search_data(self, data: Any) -> Iterator[SearchResult]:
"""turn the result json from a search into a list"""
class AbstractConnector(AbstractMinimalConnector):
"""generic book data connector"""
- def __init__(self, identifier):
+ generated_remote_link_field = ""
+
+ def __init__(self, identifier: str):
super().__init__(identifier)
# fields we want to look for in book data to copy over
# title we handle separately.
- self.book_mappings = []
+ self.book_mappings: list[Mapping] = []
+ self.author_mappings: list[Mapping] = []
- def get_or_create_book(self, remote_id):
+ def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
"""translate arbitrary json into an Activitypub dataclass"""
# first, check if we have the origin_id saved
existing = models.Edition.find_existing_by_remote_id(
remote_id
) or models.Work.find_existing_by_remote_id(remote_id)
if existing:
- if hasattr(existing, "default_edition"):
+ if hasattr(existing, "default_edition") and isinstance(
+ existing.default_edition, models.Edition
+ ):
return existing.default_edition
return existing
@@ -117,6 +177,9 @@ class AbstractConnector(AbstractMinimalConnector):
)
# this will dedupe automatically
work = work_activity.to_model(model=models.Work, overwrite=False)
+ if not work:
+ return None
+
for author in self.get_authors_from_data(work_data):
work.authors.add(author)
@@ -124,12 +187,21 @@ class AbstractConnector(AbstractMinimalConnector):
load_more_data.delay(self.connector.id, work.id)
return edition
- def get_book_data(self, remote_id): # pylint: disable=no-self-use
+ def get_book_data(self, remote_id: str) -> JsonDict: # pylint: disable=no-self-use
"""this allows connectors to override the default behavior"""
return get_data(remote_id)
- def create_edition_from_data(self, work, edition_data, instance=None):
+ def create_edition_from_data(
+ self,
+ work: models.Work,
+ edition_data: Union[str, JsonDict],
+ instance: Optional[models.Edition] = None,
+ ) -> Optional[models.Edition]:
"""if we already have the work, we're ready"""
+ if isinstance(edition_data, str):
+ # We don't expect a string here
+ return None
+
mapped_data = dict_from_mappings(edition_data, self.book_mappings)
mapped_data["work"] = work.remote_id
edition_activity = activitypub.Edition(**mapped_data)
@@ -137,6 +209,9 @@ class AbstractConnector(AbstractMinimalConnector):
model=models.Edition, overwrite=False, instance=instance
)
+ if not edition:
+ return None
+
# if we're updating an existing instance, we don't need to load authors
if instance:
return edition
@@ -153,7 +228,9 @@ class AbstractConnector(AbstractMinimalConnector):
return edition
- def get_or_create_author(self, remote_id, instance=None):
+ def get_or_create_author(
+ self, remote_id: str, instance: Optional[models.Author] = None
+ ) -> Optional[models.Author]:
"""load that author"""
if not instance:
existing = models.Author.find_existing_by_remote_id(remote_id)
@@ -173,46 +250,51 @@ class AbstractConnector(AbstractMinimalConnector):
model=models.Author, overwrite=False, instance=instance
)
- def get_remote_id_from_model(self, obj):
+ def get_remote_id_from_model(self, obj: models.BookDataModel) -> Optional[str]:
"""given the data stored, how can we look this up"""
- return getattr(obj, getattr(self, "generated_remote_link_field"))
+ remote_id: Optional[str] = getattr(obj, self.generated_remote_link_field)
+ return remote_id
- def update_author_from_remote(self, obj):
+ def update_author_from_remote(self, obj: models.Author) -> Optional[models.Author]:
"""load the remote data from this connector and add it to an existing author"""
remote_id = self.get_remote_id_from_model(obj)
+ if not remote_id:
+ return None
return self.get_or_create_author(remote_id, instance=obj)
- def update_book_from_remote(self, obj):
+ def update_book_from_remote(self, obj: models.Edition) -> Optional[models.Edition]:
"""load the remote data from this connector and add it to an existing book"""
remote_id = self.get_remote_id_from_model(obj)
+ if not remote_id:
+ return None
data = self.get_book_data(remote_id)
return self.create_edition_from_data(obj.parent_work, data, instance=obj)
@abstractmethod
- def is_work_data(self, data):
+ def is_work_data(self, data: JsonDict) -> bool:
"""differentiate works and editions"""
@abstractmethod
- def get_edition_from_work_data(self, data):
+ def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
"""every work needs at least one edition"""
@abstractmethod
- def get_work_from_edition_data(self, data):
+ def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
"""every edition needs a work"""
@abstractmethod
- def get_authors_from_data(self, data):
+ def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
"""load author data"""
@abstractmethod
- def expand_book_data(self, book):
+ def expand_book_data(self, book: models.Book) -> None:
"""get more info on a book"""
-def dict_from_mappings(data, mappings):
+def dict_from_mappings(data: JsonDict, mappings: list[Mapping]) -> JsonDict:
"""create a dict in Activitypub format, using mappings supplies by
the subclass"""
- result = {}
+ result: JsonDict = {}
for mapping in mappings:
# sometimes there are multiple mappings for one field, don't
# overwrite earlier writes in that case
@@ -222,7 +304,11 @@ def dict_from_mappings(data, mappings):
return result
-def get_data(url, params=None, timeout=10):
+def get_data(
+ url: str,
+ params: Optional[dict[str, str]] = None,
+ timeout: int = settings.QUERY_TIMEOUT,
+) -> JsonDict:
"""wrapper for request.get"""
# check if the url is blocked
raise_not_valid_url(url)
@@ -244,17 +330,26 @@ def get_data(url, params=None, timeout=10):
raise ConnectorException(err)
if not resp.ok:
- raise ConnectorException()
+ if resp.status_code == 401:
+ # this is probably an AUTHORIZED_FETCH issue
+ resp.raise_for_status()
+ else:
+ raise ConnectorException()
try:
data = resp.json()
except ValueError as err:
logger.info(err)
raise ConnectorException(err)
+ if not isinstance(data, dict):
+ raise ConnectorException("Unexpected data format")
+
return data
-def get_image(url, timeout=10):
+def get_image(
+ url: str, timeout: int = 10
+) -> Union[tuple[ContentFile[bytes], str], tuple[None, None]]:
"""wrapper for requesting an image"""
raise_not_valid_url(url)
try:
@@ -284,14 +379,19 @@ def get_image(url, timeout=10):
class Mapping:
"""associate a local database field with a field in an external dataset"""
- def __init__(self, local_field, remote_field=None, formatter=None):
+ def __init__(
+ self,
+ local_field: str,
+ remote_field: Optional[str] = None,
+ formatter: Optional[Callable[[Any], Any]] = None,
+ ):
noop = lambda x: x
self.local_field = local_field
self.remote_field = remote_field or local_field
self.formatter = formatter or noop
- def get_value(self, data):
+ def get_value(self, data: JsonDict) -> Optional[Any]:
"""pull a field from incoming json and return the formatted version"""
value = data.get(self.remote_field)
if not value:
@@ -302,7 +402,7 @@ class Mapping:
return None
-def infer_physical_format(format_text):
+def infer_physical_format(format_text: str) -> Optional[str]:
"""try to figure out what the standardized format is from the free value"""
format_text = format_text.lower()
if format_text in format_mappings:
@@ -315,8 +415,8 @@ def infer_physical_format(format_text):
return matches[0]
-def unique_physical_format(format_text):
- """only store the format if it isn't diretly in the format mappings"""
+def unique_physical_format(format_text: str) -> Optional[str]:
+ """only store the format if it isn't directly in the format mappings"""
format_text = format_text.lower()
if format_text in format_mappings:
# try a direct match, so saving this would be redundant
@@ -324,7 +424,7 @@ def unique_physical_format(format_text):
return format_text
-def maybe_isbn(query):
+def maybe_isbn(query: str) -> bool:
"""check if a query looks like an isbn"""
isbn = re.sub(r"[\W_]", "", query) # removes filler characters
# ISBNs must be numeric except an ISBN10 checkdigit can be 'X'
diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py
index e07a0b281..4064f4b4c 100644
--- a/bookwyrm/connectors/bookwyrm_connector.py
+++ b/bookwyrm/connectors/bookwyrm_connector.py
@@ -1,4 +1,7 @@
""" using another bookwyrm instance as a source of book data """
+from __future__ import annotations
+from typing import Any, Iterator
+
from bookwyrm import activitypub, models
from bookwyrm.book_search import SearchResult
from .abstract_connector import AbstractMinimalConnector
@@ -7,15 +10,19 @@ from .abstract_connector import AbstractMinimalConnector
class Connector(AbstractMinimalConnector):
"""this is basically just for search"""
- def get_or_create_book(self, remote_id):
+ def get_or_create_book(self, remote_id: str) -> models.Edition:
return activitypub.resolve_remote_id(remote_id, model=models.Edition)
- def parse_search_data(self, data, min_confidence):
+ def parse_search_data(
+ self, data: list[dict[str, Any]], min_confidence: float
+ ) -> Iterator[SearchResult]:
for search_result in data:
search_result["connector"] = self
yield SearchResult(**search_result)
- def parse_isbn_search_data(self, data):
+ def parse_isbn_search_data(
+ self, data: list[dict[str, Any]]
+ ) -> Iterator[SearchResult]:
for search_result in data:
search_result["connector"] = self
yield SearchResult(**search_result)
diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py
index 9a6f834af..444a626ba 100644
--- a/bookwyrm/connectors/connector_manager.py
+++ b/bookwyrm/connectors/connector_manager.py
@@ -1,8 +1,11 @@
""" interface with whatever connectors the app has """
+from __future__ import annotations
import asyncio
import importlib
import ipaddress
import logging
+from asyncio import Future
+from typing import Iterator, Any, Optional, Union, overload, Literal
from urllib.parse import urlparse
import aiohttp
@@ -12,8 +15,10 @@ from django.db.models import signals
from requests import HTTPError
from bookwyrm import book_search, models
-from bookwyrm.settings import SEARCH_TIMEOUT, USER_AGENT
-from bookwyrm.tasks import app, LOW
+from bookwyrm.book_search import SearchResult
+from bookwyrm.connectors import abstract_connector
+from bookwyrm.settings import SEARCH_TIMEOUT
+from bookwyrm.tasks import app, CONNECTORS
logger = logging.getLogger(__name__)
@@ -22,61 +27,46 @@ class ConnectorException(HTTPError):
"""when the connector can't do what was asked"""
-async def get_results(session, url, min_confidence, query, connector):
- """try this specific connector"""
- # pylint: disable=line-too-long
- headers = {
- "Accept": (
- 'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
- ),
- "User-Agent": USER_AGENT,
- }
- params = {"min_confidence": min_confidence}
- try:
- async with session.get(url, headers=headers, params=params) as response:
- if not response.ok:
- logger.info("Unable to connect to %s: %s", url, response.reason)
- return
-
- try:
- raw_data = await response.json()
- except aiohttp.client_exceptions.ContentTypeError as err:
- logger.exception(err)
- return
-
- return {
- "connector": connector,
- "results": connector.process_search_response(
- query, raw_data, min_confidence
- ),
- }
- except asyncio.TimeoutError:
- logger.info("Connection timed out for url: %s", url)
- except aiohttp.ClientError as err:
- logger.info(err)
-
-
-async def async_connector_search(query, items, min_confidence):
+async def async_connector_search(
+ query: str,
+ items: list[tuple[str, abstract_connector.AbstractConnector]],
+ min_confidence: float,
+) -> list[Optional[abstract_connector.ConnectorResults]]:
"""Try a number of requests simultaneously"""
timeout = aiohttp.ClientTimeout(total=SEARCH_TIMEOUT)
async with aiohttp.ClientSession(timeout=timeout) as session:
- tasks = []
+ tasks: list[Future[Optional[abstract_connector.ConnectorResults]]] = []
for url, connector in items:
tasks.append(
asyncio.ensure_future(
- get_results(session, url, min_confidence, query, connector)
+ connector.get_results(session, url, min_confidence, query)
)
)
results = await asyncio.gather(*tasks)
- return results
+ return list(results)
-def search(query, min_confidence=0.1, return_first=False):
- """find books based on arbitary keywords"""
+@overload
+def search(
+ query: str, *, min_confidence: float = 0.1, return_first: Literal[False]
+) -> list[abstract_connector.ConnectorResults]:
+ ...
+
+
+@overload
+def search(
+ query: str, *, min_confidence: float = 0.1, return_first: Literal[True]
+) -> Optional[SearchResult]:
+ ...
+
+
+def search(
+ query: str, *, min_confidence: float = 0.1, return_first: bool = False
+) -> Union[list[abstract_connector.ConnectorResults], Optional[SearchResult]]:
+ """find books based on arbitrary keywords"""
if not query:
- return []
- results = []
+ return None if return_first else []
items = []
for connector in get_connectors():
@@ -91,8 +81,12 @@ def search(query, min_confidence=0.1, return_first=False):
items.append((url, connector))
# load as many results as we can
- results = asyncio.run(async_connector_search(query, items, min_confidence))
- results = [r for r in results if r]
+ # failed requests will return None, so filter those out
+ results = [
+ r
+ for r in asyncio.run(async_connector_search(query, items, min_confidence))
+ if r
+ ]
if return_first:
# find the best result from all the responses and return that
@@ -100,11 +94,12 @@ def search(query, min_confidence=0.1, return_first=False):
all_results = sorted(all_results, key=lambda r: r.confidence, reverse=True)
return all_results[0] if all_results else None
- # failed requests will return None, so filter those out
return results
-def first_search_result(query, min_confidence=0.1):
+def first_search_result(
+ query: str, min_confidence: float = 0.1
+) -> Union[models.Edition, SearchResult, None]:
"""search until you find a result that fits"""
# try local search first
result = book_search.search(query, min_confidence=min_confidence, return_first=True)
@@ -114,13 +109,13 @@ def first_search_result(query, min_confidence=0.1):
return search(query, min_confidence=min_confidence, return_first=True) or None
-def get_connectors():
+def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
"""load all connectors"""
for info in models.Connector.objects.filter(active=True).order_by("priority").all():
yield load_connector(info)
-def get_or_create_connector(remote_id):
+def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
"""get the connector related to the object's server"""
url = urlparse(remote_id)
identifier = url.netloc
@@ -143,8 +138,8 @@ def get_or_create_connector(remote_id):
return load_connector(connector_info)
-@app.task(queue=LOW)
-def load_more_data(connector_id, book_id):
+@app.task(queue=CONNECTORS)
+def load_more_data(connector_id: str, book_id: str) -> None:
"""background the work of getting all 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id)
connector = load_connector(connector_info)
@@ -152,8 +147,10 @@ def load_more_data(connector_id, book_id):
connector.expand_book_data(book)
-@app.task(queue=LOW)
-def create_edition_task(connector_id, work_id, data):
+@app.task(queue=CONNECTORS)
+def create_edition_task(
+ connector_id: int, work_id: int, data: Union[str, abstract_connector.JsonDict]
+) -> None:
"""separate task for each of the 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id)
connector = load_connector(connector_info)
@@ -161,23 +158,31 @@ def create_edition_task(connector_id, work_id, data):
connector.create_edition_from_data(work, data)
-def load_connector(connector_info):
+def load_connector(
+ connector_info: models.Connector,
+) -> abstract_connector.AbstractConnector:
"""instantiate the connector class"""
connector = importlib.import_module(
f"bookwyrm.connectors.{connector_info.connector_file}"
)
- return connector.Connector(connector_info.identifier)
+ return connector.Connector(connector_info.identifier) # type: ignore[no-any-return]
@receiver(signals.post_save, sender="bookwyrm.FederatedServer")
# pylint: disable=unused-argument
-def create_connector(sender, instance, created, *args, **kwargs):
+def create_connector(
+ sender: Any,
+ instance: models.FederatedServer,
+ created: Any,
+ *args: Any,
+ **kwargs: Any,
+) -> None:
"""create a connector to an external bookwyrm server"""
if instance.application_type == "bookwyrm":
get_or_create_connector(f"https://{instance.server_name}")
-def raise_not_valid_url(url):
+def raise_not_valid_url(url: str) -> None:
"""do some basic reality checks on the url"""
parsed = urlparse(url)
if not parsed.scheme in ["http", "https"]:
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index df9b2e43a..c08bcdee1 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -1,9 +1,10 @@
""" inventaire data connector """
import re
+from typing import Any, Union, Optional, Iterator, Iterable
from bookwyrm import models
from bookwyrm.book_search import SearchResult
-from .abstract_connector import AbstractConnector, Mapping
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
from .abstract_connector import get_data
from .connector_manager import ConnectorException, create_edition_task
@@ -13,7 +14,7 @@ class Connector(AbstractConnector):
generated_remote_link_field = "inventaire_id"
- def __init__(self, identifier):
+ def __init__(self, identifier: str):
super().__init__(identifier)
get_first = lambda a: a[0]
@@ -60,13 +61,13 @@ class Connector(AbstractConnector):
Mapping("died", remote_field="wdt:P570", formatter=get_first),
] + shared_mappings
- def get_remote_id(self, value):
+ def get_remote_id(self, value: str) -> str:
"""convert an id/uri into a url"""
return f"{self.books_url}?action=by-uris&uris={value}"
- def get_book_data(self, remote_id):
+ def get_book_data(self, remote_id: str) -> JsonDict:
data = get_data(remote_id)
- extracted = list(data.get("entities").values())
+ extracted = list(data.get("entities", {}).values())
try:
data = extracted[0]
except (KeyError, IndexError):
@@ -74,10 +75,16 @@ class Connector(AbstractConnector):
# flatten the data so that images, uri, and claims are on the same level
return {
**data.get("claims", {}),
- **{k: data.get(k) for k in ["uri", "image", "labels", "sitelinks", "type"]},
+ **{
+ k: data.get(k)
+ for k in ["uri", "image", "labels", "sitelinks", "type"]
+ if k in data
+ },
}
- def parse_search_data(self, data, min_confidence):
+ def parse_search_data(
+ self, data: JsonDict, min_confidence: float
+ ) -> Iterator[SearchResult]:
for search_result in data.get("results", []):
images = search_result.get("image")
cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None
@@ -96,8 +103,8 @@ class Connector(AbstractConnector):
connector=self,
)
- def parse_isbn_search_data(self, data):
- """got some daaaata"""
+ def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
+ """got some data"""
results = data.get("entities")
if not results:
return
@@ -114,35 +121,44 @@ class Connector(AbstractConnector):
connector=self,
)
- def is_work_data(self, data):
+ def is_work_data(self, data: JsonDict) -> bool:
return data.get("type") == "work"
- def load_edition_data(self, work_uri):
+ def load_edition_data(self, work_uri: str) -> JsonDict:
"""get a list of editions for a work"""
# pylint: disable=line-too-long
url = f"{self.books_url}?action=reverse-claims&property=wdt:P629&value={work_uri}&sort=true"
return get_data(url)
- def get_edition_from_work_data(self, data):
- data = self.load_edition_data(data.get("uri"))
+ def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
+ work_uri = data.get("uri")
+ if not work_uri:
+ raise ConnectorException("Invalid URI")
+ data = self.load_edition_data(work_uri)
try:
uri = data.get("uris", [])[0]
except IndexError:
raise ConnectorException("Invalid book data")
return self.get_book_data(self.get_remote_id(uri))
- def get_work_from_edition_data(self, data):
- uri = data.get("wdt:P629", [None])[0]
+ def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
+ try:
+ uri = data.get("wdt:P629", [])[0]
+ except IndexError:
+ raise ConnectorException("Invalid book data")
+
if not uri:
raise ConnectorException("Invalid book data")
return self.get_book_data(self.get_remote_id(uri))
- def get_authors_from_data(self, data):
+ def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
authors = data.get("wdt:P50", [])
for author in authors:
- yield self.get_or_create_author(self.get_remote_id(author))
+ model = self.get_or_create_author(self.get_remote_id(author))
+ if model:
+ yield model
- def expand_book_data(self, book):
+ def expand_book_data(self, book: models.Book) -> None:
work = book
# go from the edition to the work, if necessary
if isinstance(book, models.Edition):
@@ -154,36 +170,45 @@ class Connector(AbstractConnector):
# who knows, man
return
- for edition_uri in edition_options.get("uris"):
+ for edition_uri in edition_options.get("uris", []):
remote_id = self.get_remote_id(edition_uri)
create_edition_task.delay(self.connector.id, work.id, remote_id)
- def create_edition_from_data(self, work, edition_data, instance=None):
+ def create_edition_from_data(
+ self,
+ work: models.Work,
+ edition_data: Union[str, JsonDict],
+ instance: Optional[models.Edition] = None,
+ ) -> Optional[models.Edition]:
"""pass in the url as data and then call the version in abstract connector"""
if isinstance(edition_data, str):
try:
edition_data = self.get_book_data(edition_data)
except ConnectorException:
# who, indeed, knows
- return
- super().create_edition_from_data(work, edition_data, instance=instance)
+ return None
+ return super().create_edition_from_data(work, edition_data, instance=instance)
- def get_cover_url(self, cover_blob, *_):
+ def get_cover_url(
+ self, cover_blob: Union[list[JsonDict], JsonDict], *_: Any
+ ) -> Optional[str]:
"""format the relative cover url into an absolute one:
{"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"}
"""
# covers may or may not be a list
- if isinstance(cover_blob, list) and len(cover_blob) > 0:
+ if isinstance(cover_blob, list):
+ if len(cover_blob) == 0:
+ return None
cover_blob = cover_blob[0]
cover_id = cover_blob.get("url")
- if not cover_id:
+ if not isinstance(cover_id, str):
return None
# cover may or may not be an absolute url already
if re.match(r"^http", cover_id):
return cover_id
return f"{self.covers_url}{cover_id}"
- def resolve_keys(self, keys):
+ def resolve_keys(self, keys: Iterable[str]) -> list[str]:
"""cool, it's "wd:Q3156592" now what the heck does that mean"""
results = []
for uri in keys:
@@ -191,10 +216,10 @@ class Connector(AbstractConnector):
data = self.get_book_data(self.get_remote_id(uri))
except ConnectorException:
continue
- results.append(get_language_code(data.get("labels")))
+ results.append(get_language_code(data.get("labels", {})))
return results
- def get_description(self, links):
+ def get_description(self, links: JsonDict) -> str:
"""grab an extracted excerpt from wikipedia"""
link = links.get("enwiki")
if not link:
@@ -204,15 +229,15 @@ class Connector(AbstractConnector):
data = get_data(url)
except ConnectorException:
return ""
- return data.get("extract")
+ return data.get("extract", "")
- def get_remote_id_from_model(self, obj):
+ def get_remote_id_from_model(self, obj: models.BookDataModel) -> str:
"""use get_remote_id to figure out the link from a model obj"""
remote_id_value = obj.inventaire_id
return self.get_remote_id(remote_id_value)
-def get_language_code(options, code="en"):
+def get_language_code(options: JsonDict, code: str = "en") -> Any:
"""when there are a bunch of translation but we need a single field"""
result = options.get(code)
if result:
diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py
index 0fd786660..4dc6d6ac1 100644
--- a/bookwyrm/connectors/openlibrary.py
+++ b/bookwyrm/connectors/openlibrary.py
@@ -1,9 +1,13 @@
""" openlibrary data connector """
import re
+from typing import Any, Optional, Union, Iterator, Iterable
+
+from markdown import markdown
from bookwyrm import models
from bookwyrm.book_search import SearchResult
-from .abstract_connector import AbstractConnector, Mapping
+from bookwyrm.utils.sanitizer import clean
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
from .abstract_connector import get_data, infer_physical_format, unique_physical_format
from .connector_manager import ConnectorException, create_edition_task
from .openlibrary_languages import languages
@@ -14,7 +18,7 @@ class Connector(AbstractConnector):
generated_remote_link_field = "openlibrary_link"
- def __init__(self, identifier):
+ def __init__(self, identifier: str):
super().__init__(identifier)
get_first = lambda a, *args: a[0]
@@ -94,14 +98,14 @@ class Connector(AbstractConnector):
Mapping("inventaire_id", remote_field="links", formatter=get_inventaire_id),
]
- def get_book_data(self, remote_id):
+ def get_book_data(self, remote_id: str) -> JsonDict:
data = get_data(remote_id)
if data.get("type", {}).get("key") == "/type/redirect":
- remote_id = self.base_url + data.get("location")
+ remote_id = self.base_url + data.get("location", "")
return get_data(remote_id)
return data
- def get_remote_id_from_data(self, data):
+ def get_remote_id_from_data(self, data: JsonDict) -> str:
"""format a url from an openlibrary id field"""
try:
key = data["key"]
@@ -109,10 +113,10 @@ class Connector(AbstractConnector):
raise ConnectorException("Invalid book data")
return f"{self.books_url}{key}"
- def is_work_data(self, data):
+ def is_work_data(self, data: JsonDict) -> bool:
return bool(re.match(r"^[\/\w]+OL\d+W$", data["key"]))
- def get_edition_from_work_data(self, data):
+ def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
try:
key = data["key"]
except KeyError:
@@ -124,7 +128,7 @@ class Connector(AbstractConnector):
raise ConnectorException("No editions for work")
return edition
- def get_work_from_edition_data(self, data):
+ def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
try:
key = data["works"][0]["key"]
except (IndexError, KeyError):
@@ -132,7 +136,7 @@ class Connector(AbstractConnector):
url = f"{self.books_url}{key}"
return self.get_book_data(url)
- def get_authors_from_data(self, data):
+ def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
"""parse author json and load or create authors"""
for author_blob in data.get("authors", []):
author_blob = author_blob.get("author", author_blob)
@@ -144,7 +148,7 @@ class Connector(AbstractConnector):
continue
yield author
- def get_cover_url(self, cover_blob, size="L"):
+ def get_cover_url(self, cover_blob: list[str], size: str = "L") -> Optional[str]:
"""ask openlibrary for the cover"""
if not cover_blob:
return None
@@ -152,8 +156,10 @@ class Connector(AbstractConnector):
image_name = f"{cover_id}-{size}.jpg"
return f"{self.covers_url}/b/id/{image_name}"
- def parse_search_data(self, data, min_confidence):
- for idx, search_result in enumerate(data.get("docs")):
+ def parse_search_data(
+ self, data: JsonDict, min_confidence: float
+ ) -> Iterator[SearchResult]:
+ for idx, search_result in enumerate(data.get("docs", [])):
# build the remote id from the openlibrary key
key = self.books_url + search_result["key"]
author = search_result.get("author_name") or ["Unknown"]
@@ -174,7 +180,7 @@ class Connector(AbstractConnector):
confidence=confidence,
)
- def parse_isbn_search_data(self, data):
+ def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
for search_result in list(data.values()):
# build the remote id from the openlibrary key
key = self.books_url + search_result["key"]
@@ -188,12 +194,12 @@ class Connector(AbstractConnector):
year=search_result.get("publish_date"),
)
- def load_edition_data(self, olkey):
+ def load_edition_data(self, olkey: str) -> JsonDict:
"""query openlibrary for editions of a work"""
url = f"{self.books_url}/works/{olkey}/editions"
return self.get_book_data(url)
- def expand_book_data(self, book):
+ def expand_book_data(self, book: models.Book) -> None:
work = book
# go from the edition to the work, if necessary
if isinstance(book, models.Edition):
@@ -206,14 +212,14 @@ class Connector(AbstractConnector):
# who knows, man
return
- for edition_data in edition_options.get("entries"):
+ for edition_data in edition_options.get("entries", []):
# does this edition have ANY interesting data?
if ignore_edition(edition_data):
continue
create_edition_task.delay(self.connector.id, work.id, edition_data)
-def ignore_edition(edition_data):
+def ignore_edition(edition_data: JsonDict) -> bool:
"""don't load a million editions that have no metadata"""
# an isbn, we love to see it
if edition_data.get("isbn_13") or edition_data.get("isbn_10"):
@@ -232,19 +238,30 @@ def ignore_edition(edition_data):
return True
-def get_description(description_blob):
+def get_description(description_blob: Union[JsonDict, str]) -> str:
"""descriptions can be a string or a dict"""
if isinstance(description_blob, dict):
- return description_blob.get("value")
- return description_blob
+ description = markdown(description_blob.get("value", ""))
+ else:
+ description = markdown(description_blob)
+
+ if (
+ description.startswith("")
+ and description.endswith("
")
+ and description.count("") == 1
+ ):
+ # If there is just one
tag and it is around the text remove it
+ return description[len("
") : -len("
")].strip()
+
+ return clean(description)
-def get_openlibrary_key(key):
+def get_openlibrary_key(key: str) -> str:
"""convert /books/OL27320736M into OL27320736M"""
return key.split("/")[-1]
-def get_languages(language_blob):
+def get_languages(language_blob: Iterable[JsonDict]) -> list[Optional[str]]:
"""/language/eng -> English"""
langs = []
for lang in language_blob:
@@ -252,14 +269,14 @@ def get_languages(language_blob):
return langs
-def get_dict_field(blob, field_name):
+def get_dict_field(blob: Optional[JsonDict], field_name: str) -> Optional[Any]:
"""extract the isni from the remote id data for the author"""
if not blob or not isinstance(blob, dict):
return None
return blob.get(field_name)
-def get_wikipedia_link(links):
+def get_wikipedia_link(links: list[Any]) -> Optional[str]:
"""extract wikipedia links"""
if not isinstance(links, list):
return None
@@ -272,7 +289,7 @@ def get_wikipedia_link(links):
return None
-def get_inventaire_id(links):
+def get_inventaire_id(links: list[Any]) -> Optional[str]:
"""extract and format inventaire ids"""
if not isinstance(links, list):
return None
@@ -282,11 +299,13 @@ def get_inventaire_id(links):
continue
if link.get("title") == "inventaire.io":
iv_link = link.get("url")
+ if not isinstance(iv_link, str):
+ return None
return iv_link.split("/")[-1]
return None
-def pick_default_edition(options):
+def pick_default_edition(options: list[JsonDict]) -> Optional[JsonDict]:
"""favor physical copies with covers in english"""
if not options:
return None
diff --git a/bookwyrm/emailing.py b/bookwyrm/emailing.py
index e767d5374..5e08ebba1 100644
--- a/bookwyrm/emailing.py
+++ b/bookwyrm/emailing.py
@@ -3,7 +3,7 @@ from django.core.mail import EmailMultiAlternatives
from django.template.loader import get_template
from bookwyrm import models, settings
-from bookwyrm.tasks import app, HIGH
+from bookwyrm.tasks import app, EMAIL
from bookwyrm.settings import DOMAIN
@@ -18,6 +18,12 @@ def email_data():
}
+def test_email(user):
+ """Just an admin checking if emails are sending"""
+ data = email_data()
+ send_email(user.email, *format_email("test", data))
+
+
def email_confirmation_email(user):
"""newly registered users confirm email address"""
data = email_data()
@@ -38,7 +44,7 @@ def password_reset_email(reset_code):
data = email_data()
data["reset_link"] = reset_code.link
data["user"] = reset_code.user.display_name
- send_email.delay(reset_code.user.email, *format_email("password_reset", data))
+ send_email(reset_code.user.email, *format_email("password_reset", data))
def moderation_report_email(report):
@@ -48,6 +54,7 @@ def moderation_report_email(report):
if report.user:
data["reportee"] = report.user.localname or report.user.username
data["report_link"] = report.remote_id
+ data["link_domain"] = report.links.exists()
for admin in models.User.objects.filter(
groups__name__in=["admin", "moderator"]
@@ -68,7 +75,7 @@ def format_email(email_name, data):
return (subject, html_content, text_content)
-@app.task(queue=HIGH)
+@app.task(queue=EMAIL)
def send_email(recipient, subject, html_content, text_content):
"""use a task to send the email"""
email = EmailMultiAlternatives(
diff --git a/bookwyrm/forms/admin.py b/bookwyrm/forms/admin.py
index ae15e011b..72f50ccb8 100644
--- a/bookwyrm/forms/admin.py
+++ b/bookwyrm/forms/admin.py
@@ -15,7 +15,7 @@ from .custom_form import CustomForm, StyledForm
# pylint: disable=missing-class-docstring
class ExpiryWidget(widgets.Select):
def value_from_datadict(self, data, files, name):
- """human-readable exiration time buckets"""
+ """human-readable expiration time buckets"""
selected_string = super().value_from_datadict(data, files, name)
if selected_string == "day":
@@ -55,11 +55,46 @@ class CreateInviteForm(CustomForm):
class SiteForm(CustomForm):
class Meta:
model = models.SiteSettings
- exclude = ["admin_code", "install_mode"]
+ fields = [
+ "name",
+ "instance_tagline",
+ "instance_description",
+ "instance_short_description",
+ "default_theme",
+ "code_of_conduct",
+ "privacy_policy",
+ "impressum",
+ "show_impressum",
+ "logo",
+ "logo_small",
+ "favicon",
+ "support_link",
+ "support_title",
+ "admin_email",
+ "footer_item",
+ ]
widgets = {
"instance_short_description": forms.TextInput(
attrs={"aria-describedby": "desc_instance_short_description"}
),
+ }
+
+
+class RegistrationForm(CustomForm):
+ class Meta:
+ model = models.SiteSettings
+ fields = [
+ "allow_registration",
+ "allow_invite_requests",
+ "registration_closed_text",
+ "invite_request_text",
+ "invite_request_question",
+ "invite_question_text",
+ "require_confirm_email",
+ "default_user_auth_group",
+ ]
+
+ widgets = {
"require_confirm_email": forms.CheckboxInput(
attrs={"aria-describedby": "desc_require_confirm_email"}
),
@@ -69,6 +104,23 @@ class SiteForm(CustomForm):
}
+class RegistrationLimitedForm(CustomForm):
+ class Meta:
+ model = models.SiteSettings
+ fields = [
+ "registration_closed_text",
+ "invite_request_text",
+ "invite_request_question",
+ "invite_question_text",
+ ]
+
+ widgets = {
+ "invite_request_text": forms.Textarea(
+ attrs={"aria-describedby": "desc_invite_request_text"}
+ ),
+ }
+
+
class ThemeForm(CustomForm):
class Meta:
model = models.Theme
diff --git a/bookwyrm/forms/author.py b/bookwyrm/forms/author.py
index ca59426de..5b54a07b5 100644
--- a/bookwyrm/forms/author.py
+++ b/bookwyrm/forms/author.py
@@ -15,12 +15,14 @@ class AuthorForm(CustomForm):
"aliases",
"bio",
"wikipedia_link",
+ "website",
"born",
"died",
"openlibrary_key",
"inventaire_id",
"librarything_key",
"goodreads_key",
+ "isfdb",
"isni",
]
widgets = {
@@ -30,10 +32,11 @@ class AuthorForm(CustomForm):
"wikipedia_link": forms.TextInput(
attrs={"aria-describedby": "desc_wikipedia_link"}
),
+ "website": forms.TextInput(attrs={"aria-describedby": "desc_website"}),
"born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
"died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),
- "oepnlibrary_key": forms.TextInput(
- attrs={"aria-describedby": "desc_oepnlibrary_key"}
+ "openlibrary_key": forms.TextInput(
+ attrs={"aria-describedby": "desc_openlibrary_key"}
),
"inventaire_id": forms.TextInput(
attrs={"aria-describedby": "desc_inventaire_id"}
diff --git a/bookwyrm/forms/books.py b/bookwyrm/forms/books.py
index 9b3c84010..4885dc063 100644
--- a/bookwyrm/forms/books.py
+++ b/bookwyrm/forms/books.py
@@ -18,22 +18,37 @@ class CoverForm(CustomForm):
class EditionForm(CustomForm):
class Meta:
model = models.Edition
- exclude = [
- "remote_id",
- "origin_id",
- "created_date",
- "updated_date",
- "edition_rank",
- "authors",
- "parent_work",
- "shelves",
- "connector",
- "search_vector",
- "links",
- "file_links",
+ fields = [
+ "title",
+ "sort_title",
+ "subtitle",
+ "description",
+ "series",
+ "series_number",
+ "languages",
+ "subjects",
+ "publishers",
+ "first_published_date",
+ "published_date",
+ "cover",
+ "physical_format",
+ "physical_format_detail",
+ "pages",
+ "isbn_13",
+ "isbn_10",
+ "openlibrary_key",
+ "inventaire_id",
+ "goodreads_key",
+ "oclc_number",
+ "asin",
+ "aasin",
+ "isfdb",
]
widgets = {
"title": forms.TextInput(attrs={"aria-describedby": "desc_title"}),
+ "sort_title": forms.TextInput(
+ attrs={"aria-describedby": "desc_sort_title"}
+ ),
"subtitle": forms.TextInput(attrs={"aria-describedby": "desc_subtitle"}),
"description": forms.Textarea(
attrs={"aria-describedby": "desc_description"}
@@ -73,10 +88,15 @@ class EditionForm(CustomForm):
"inventaire_id": forms.TextInput(
attrs={"aria-describedby": "desc_inventaire_id"}
),
+ "goodreads_key": forms.TextInput(
+ attrs={"aria-describedby": "desc_goodreads_key"}
+ ),
"oclc_number": forms.TextInput(
attrs={"aria-describedby": "desc_oclc_number"}
),
"ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}),
+ "AASIN": forms.TextInput(attrs={"aria-describedby": "desc_AASIN"}),
+ "isfdb": forms.TextInput(attrs={"aria-describedby": "desc_isfdb"}),
}
@@ -91,6 +111,7 @@ class EditionFromWorkForm(CustomForm):
model = models.Work
fields = [
"title",
+ "sort_title",
"subtitle",
"authors",
"description",
diff --git a/bookwyrm/forms/landing.py b/bookwyrm/forms/landing.py
index bd9884bc3..1da4fc4f1 100644
--- a/bookwyrm/forms/landing.py
+++ b/bookwyrm/forms/landing.py
@@ -8,6 +8,7 @@ import pyotp
from bookwyrm import models
from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import TWO_FACTOR_LOGIN_VALIDITY_WINDOW
from .custom_form import CustomForm
@@ -108,7 +109,7 @@ class Confirm2FAForm(CustomForm):
otp = self.data.get("otp")
totp = pyotp.TOTP(self.instance.otp_secret)
- if not totp.verify(otp):
+ if not totp.verify(otp, valid_window=TWO_FACTOR_LOGIN_VALIDITY_WINDOW):
if self.instance.hotp_secret:
# maybe it's a backup code?
diff --git a/bookwyrm/forms/links.py b/bookwyrm/forms/links.py
index de229bc2d..d2fd5f116 100644
--- a/bookwyrm/forms/links.py
+++ b/bookwyrm/forms/links.py
@@ -36,13 +36,16 @@ class FileLinkForm(CustomForm):
"This domain is blocked. Please contact your administrator if you think this is an error."
),
)
- elif models.FileLink.objects.filter(
+ if (
+ not self.instance
+ and models.FileLink.objects.filter(
url=url, book=book, filetype=filetype
- ).exists():
- # pylint: disable=line-too-long
- self.add_error(
- "url",
- _(
- "This link with file type has already been added for this book. If it is not visible, the domain is still pending."
- ),
- )
+ ).exists()
+ ):
+ # pylint: disable=line-too-long
+ self.add_error(
+ "url",
+ _(
+ "This link with file type has already been added for this book. If it is not visible, the domain is still pending."
+ ),
+ )
diff --git a/bookwyrm/forms/lists.py b/bookwyrm/forms/lists.py
index 647db3bfe..f5008baa3 100644
--- a/bookwyrm/forms/lists.py
+++ b/bookwyrm/forms/lists.py
@@ -24,7 +24,7 @@ class SortListForm(forms.Form):
sort_by = ChoiceField(
choices=(
("order", _("List Order")),
- ("title", _("Book Title")),
+ ("sort_title", _("Book Title")),
("rating", _("Rating")),
),
label=_("Sort By"),
diff --git a/bookwyrm/forms/status.py b/bookwyrm/forms/status.py
index 0800166bf..b562595ee 100644
--- a/bookwyrm/forms/status.py
+++ b/bookwyrm/forms/status.py
@@ -53,6 +53,7 @@ class QuotationForm(CustomForm):
"sensitive",
"privacy",
"position",
+ "endposition",
"position_mode",
]
diff --git a/bookwyrm/importers/calibre_import.py b/bookwyrm/importers/calibre_import.py
index 5426e9333..5c22a539d 100644
--- a/bookwyrm/importers/calibre_import.py
+++ b/bookwyrm/importers/calibre_import.py
@@ -1,4 +1,6 @@
""" handle reading a csv from calibre """
+from typing import Any, Optional
+
from bookwyrm.models import Shelf
from . import Importer
@@ -9,7 +11,7 @@ class CalibreImporter(Importer):
service = "Calibre"
- def __init__(self, *args, **kwargs):
+ def __init__(self, *args: Any, **kwargs: Any):
# Add timestamp to row_mappings_guesses for date_added to avoid
# integrity error
row_mappings_guesses = []
@@ -23,6 +25,6 @@ class CalibreImporter(Importer):
self.row_mappings_guesses = row_mappings_guesses
super().__init__(*args, **kwargs)
- def get_shelf(self, normalized_row):
+ def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
# Calibre export does not indicate which shelf to use. Use a default one for now
return Shelf.TO_READ
diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py
index a3cfba198..5b3192fa5 100644
--- a/bookwyrm/importers/importer.py
+++ b/bookwyrm/importers/importer.py
@@ -1,7 +1,10 @@
""" handle reading a csv from an external service, defaults are from Goodreads """
import csv
+from datetime import timedelta
+from typing import Iterable, Optional
+
from django.utils import timezone
-from bookwyrm.models import ImportJob, ImportItem
+from bookwyrm.models import ImportJob, ImportItem, SiteSettings, User
class Importer:
@@ -16,8 +19,8 @@ class Importer:
("id", ["id", "book id"]),
("title", ["title"]),
("authors", ["author", "authors", "primary author"]),
- ("isbn_10", ["isbn10", "isbn"]),
- ("isbn_13", ["isbn13", "isbn", "isbns"]),
+ ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
+ ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
("review_name", ["review name"]),
("review_body", ["my review", "review"]),
@@ -33,26 +36,48 @@ class Importer:
"reading": ["currently-reading", "reading", "currently reading"],
}
- def create_job(self, user, csv_file, include_reviews, privacy):
+ # pylint: disable=too-many-locals
+ def create_job(
+ self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str
+ ) -> ImportJob:
"""check over a csv and creates a database entry for the job"""
csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
- rows = enumerate(list(csv_reader))
+ rows = list(csv_reader)
+ if len(rows) < 1:
+ raise ValueError("CSV file is empty")
+
+ mappings = (
+ self.create_row_mappings(list(fieldnames))
+ if (fieldnames := csv_reader.fieldnames)
+ else {}
+ )
+
job = ImportJob.objects.create(
user=user,
include_reviews=include_reviews,
privacy=privacy,
- mappings=self.create_row_mappings(csv_reader.fieldnames),
+ mappings=mappings,
source=self.service,
)
- for index, entry in rows:
+ enforce_limit, allowed_imports = self.get_import_limit(user)
+ if enforce_limit and allowed_imports <= 0:
+ job.complete_job()
+ return job
+ for index, entry in enumerate(rows):
+ if enforce_limit and index >= allowed_imports:
+ break
self.create_item(job, index, entry)
return job
- def update_legacy_job(self, job):
+ def update_legacy_job(self, job: ImportJob) -> None:
"""patch up a job that was in the old format"""
items = job.items
- headers = list(items.first().data.keys())
+ first_item = items.first()
+ if first_item is None:
+ return
+
+ headers = list(first_item.data.keys())
job.mappings = self.create_row_mappings(headers)
job.updated_date = timezone.now()
job.save()
@@ -63,24 +88,24 @@ class Importer:
item.normalized_data = normalized
item.save()
- def create_row_mappings(self, headers):
+ def create_row_mappings(self, headers: list[str]) -> dict[str, Optional[str]]:
"""guess what the headers mean"""
mappings = {}
for (key, guesses) in self.row_mappings_guesses:
- value = [h for h in headers if h.lower() in guesses]
- value = value[0] if len(value) else None
+ values = [h for h in headers if h.lower() in guesses]
+ value = values[0] if len(values) else None
if value:
headers.remove(value)
mappings[key] = value
return mappings
- def create_item(self, job, index, data):
+ def create_item(self, job: ImportJob, index: int, data: dict[str, str]) -> None:
"""creates and saves an import item"""
normalized = self.normalize_row(data, job.mappings)
normalized["shelf"] = self.get_shelf(normalized)
ImportItem(job=job, index=index, data=data, normalized_data=normalized).save()
- def get_shelf(self, normalized_row):
+ def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
"""determine which shelf to use"""
shelf_name = normalized_row.get("shelf")
if not shelf_name:
@@ -91,11 +116,35 @@ class Importer:
]
return shelf[0] if shelf else None
- def normalize_row(self, entry, mappings): # pylint: disable=no-self-use
+ # pylint: disable=no-self-use
+ def normalize_row(
+ self, entry: dict[str, str], mappings: dict[str, Optional[str]]
+ ) -> dict[str, Optional[str]]:
"""use the dataclass to create the formatted row of data"""
- return {k: entry.get(v) for k, v in mappings.items()}
+ return {k: entry.get(v) if v else None for k, v in mappings.items()}
- def create_retry_job(self, user, original_job, items):
+ # pylint: disable=no-self-use
+ def get_import_limit(self, user: User) -> tuple[int, int]:
+ """check if import limit is set and return how many imports are left"""
+ site_settings = SiteSettings.objects.get()
+ import_size_limit = site_settings.import_size_limit
+ import_limit_reset = site_settings.import_limit_reset
+ enforce_limit = import_size_limit and import_limit_reset
+ allowed_imports = 0
+
+ if enforce_limit:
+ time_range = timezone.now() - timedelta(days=import_limit_reset)
+ import_jobs = ImportJob.objects.filter(
+ user=user, created_date__gte=time_range
+ )
+ # pylint: disable=consider-using-generator
+ imported_books = sum([job.successful_item_count for job in import_jobs])
+ allowed_imports = import_size_limit - imported_books
+ return enforce_limit, allowed_imports
+
+ def create_retry_job(
+ self, user: User, original_job: ImportJob, items: list[ImportItem]
+ ) -> ImportJob:
"""retry items that didn't import"""
job = ImportJob.objects.create(
user=user,
@@ -106,7 +155,13 @@ class Importer:
mappings=original_job.mappings,
retry=True,
)
- for item in items:
+ enforce_limit, allowed_imports = self.get_import_limit(user)
+ if enforce_limit and allowed_imports <= 0:
+ job.complete_job()
+ return job
+ for index, item in enumerate(items):
+ if enforce_limit and index >= allowed_imports:
+ break
# this will re-normalize the raw data
self.create_item(job, item.index, item.data)
return job
diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py
index c6833547d..145657ba0 100644
--- a/bookwyrm/importers/librarything_import.py
+++ b/bookwyrm/importers/librarything_import.py
@@ -1,11 +1,16 @@
""" handle reading a tsv from librarything """
import re
+from typing import Optional
from bookwyrm.models import Shelf
from . import Importer
+def _remove_brackets(value: Optional[str]) -> Optional[str]:
+ return re.sub(r"\[|\]", "", value) if value else None
+
+
class LibrarythingImporter(Importer):
"""csv downloads from librarything"""
@@ -13,16 +18,19 @@ class LibrarythingImporter(Importer):
delimiter = "\t"
encoding = "ISO-8859-1"
- def normalize_row(self, entry, mappings): # pylint: disable=no-self-use
+ def normalize_row(
+ self, entry: dict[str, str], mappings: dict[str, Optional[str]]
+ ) -> dict[str, Optional[str]]: # pylint: disable=no-self-use
"""use the dataclass to create the formatted row of data"""
- remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None
- normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()}
- isbn_13 = normalized.get("isbn_13")
- isbn_13 = isbn_13.split(", ") if isbn_13 else []
- normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 0 else None
+ normalized = {
+ k: _remove_brackets(entry.get(v) if v else None)
+ for k, v in mappings.items()
+ }
+ isbn_13 = value.split(", ") if (value := normalized.get("isbn_13")) else []
+ normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 1 else None
return normalized
- def get_shelf(self, normalized_row):
+ def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
if normalized_row["date_finished"]:
return Shelf.READ_FINISHED
if normalized_row["date_started"]:
diff --git a/bookwyrm/importers/openlibrary_import.py b/bookwyrm/importers/openlibrary_import.py
index ef1030609..6a954ed3c 100644
--- a/bookwyrm/importers/openlibrary_import.py
+++ b/bookwyrm/importers/openlibrary_import.py
@@ -1,4 +1,6 @@
""" handle reading a csv from openlibrary"""
+from typing import Any
+
from . import Importer
@@ -7,7 +9,7 @@ class OpenLibraryImporter(Importer):
service = "OpenLibrary"
- def __init__(self, *args, **kwargs):
+ def __init__(self, *args: Any, **kwargs: Any):
self.row_mappings_guesses.append(("openlibrary_key", ["edition id"]))
self.row_mappings_guesses.append(("openlibrary_work_key", ["work id"]))
super().__init__(*args, **kwargs)
diff --git a/bookwyrm/isbn/RangeMessage.xml b/bookwyrm/isbn/RangeMessage.xml
new file mode 100644
index 000000000..619cf1ff7
--- /dev/null
+++ b/bookwyrm/isbn/RangeMessage.xml
@@ -0,0 +1,7904 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+]>
+
+ International ISBN Agency
+ fa1a5bb4-9703-4910-bd34-2ffe0ae46c45
+ Sat, 22 Jul 2023 02:00:37 BST
+
+
+ 978
+ International ISBN Agency
+
+
+ 0000000-5999999
+ 1
+
+
+ 6000000-6499999
+ 3
+
+
+ 6500000-6599999
+ 2
+
+
+ 6600000-6999999
+ 0
+
+
+ 7000000-7999999
+ 1
+
+
+ 8000000-9499999
+ 2
+
+
+ 9500000-9899999
+ 3
+
+
+ 9900000-9989999
+ 4
+
+
+ 9990000-9999999
+ 5
+
+
+
+
+ 979
+ International ISBN Agency
+
+
+ 0000000-0999999
+ 0
+
+
+ 1000000-1299999
+ 2
+
+
+ 1300000-7999999
+ 0
+
+
+ 8000000-8999999
+ 1
+
+
+ 9000000-9999999
+ 0
+
+
+
+
+
+
+ 978-0
+ English language
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-2279999
+ 3
+
+
+ 2280000-2289999
+ 4
+
+
+ 2290000-3689999
+ 3
+
+
+ 3690000-3699999
+ 4
+
+
+ 3700000-6389999
+ 3
+
+
+ 6390000-6397999
+ 4
+
+
+ 6398000-6399999
+ 7
+
+
+ 6400000-6449999
+ 3
+
+
+ 6450000-6459999
+ 7
+
+
+ 6460000-6479999
+ 3
+
+
+ 6480000-6489999
+ 7
+
+
+ 6490000-6549999
+ 3
+
+
+ 6550000-6559999
+ 4
+
+
+ 6560000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9499999
+ 6
+
+
+ 9500000-9999999
+ 7
+
+
+
+
+ 978-1
+ English language
+
+
+ 0000000-0099999
+ 3
+
+
+ 0100000-0299999
+ 2
+
+
+ 0300000-0349999
+ 3
+
+
+ 0350000-0399999
+ 4
+
+
+ 0400000-0499999
+ 3
+
+
+ 0500000-0699999
+ 2
+
+
+ 0700000-0999999
+ 4
+
+
+ 1000000-3979999
+ 3
+
+
+ 3980000-5499999
+ 4
+
+
+ 5500000-6499999
+ 5
+
+
+ 6500000-6799999
+ 4
+
+
+ 6800000-6859999
+ 5
+
+
+ 6860000-7139999
+ 4
+
+
+ 7140000-7169999
+ 3
+
+
+ 7170000-7319999
+ 4
+
+
+ 7320000-7399999
+ 7
+
+
+ 7400000-7749999
+ 5
+
+
+ 7750000-7753999
+ 7
+
+
+ 7754000-7763999
+ 5
+
+
+ 7764000-7764999
+ 7
+
+
+ 7765000-7769999
+ 5
+
+
+ 7770000-7782999
+ 7
+
+
+ 7783000-7899999
+ 5
+
+
+ 7900000-7999999
+ 4
+
+
+ 8000000-8004999
+ 5
+
+
+ 8005000-8049999
+ 5
+
+
+ 8050000-8379999
+ 5
+
+
+ 8380000-8384999
+ 7
+
+
+ 8385000-8671999
+ 5
+
+
+ 8672000-8675999
+ 4
+
+
+ 8676000-8697999
+ 5
+
+
+ 8698000-9159999
+ 6
+
+
+ 9160000-9165059
+ 7
+
+
+ 9165060-9168699
+ 6
+
+
+ 9168700-9169079
+ 7
+
+
+ 9169080-9195999
+ 6
+
+
+ 9196000-9196549
+ 7
+
+
+ 9196550-9729999
+ 6
+
+
+ 9730000-9877999
+ 4
+
+
+ 9878000-9911499
+ 6
+
+
+ 9911500-9911999
+ 7
+
+
+ 9912000-9989899
+ 6
+
+
+ 9989900-9999999
+ 7
+
+
+
+
+ 978-2
+ French language
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-3499999
+ 3
+
+
+ 3500000-3999999
+ 5
+
+
+ 4000000-4869999
+ 3
+
+
+ 4870000-4949999
+ 6
+
+
+ 4950000-4959999
+ 3
+
+
+ 4960000-4966999
+ 4
+
+
+ 4967000-4969999
+ 5
+
+
+ 4970000-5279999
+ 3
+
+
+ 5280000-5299999
+ 4
+
+
+ 5300000-6999999
+ 3
+
+
+ 7000000-8399999
+ 4
+
+
+ 8400000-8999999
+ 5
+
+
+ 9000000-9197999
+ 6
+
+
+ 9198000-9198099
+ 5
+
+
+ 9198100-9199429
+ 6
+
+
+ 9199430-9199689
+ 7
+
+
+ 9199690-9499999
+ 6
+
+
+ 9500000-9999999
+ 7
+
+
+
+
+ 978-3
+ German language
+
+
+ 0000000-0299999
+ 2
+
+
+ 0300000-0339999
+ 3
+
+
+ 0340000-0369999
+ 4
+
+
+ 0370000-0399999
+ 5
+
+
+ 0400000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9499999
+ 6
+
+
+ 9500000-9539999
+ 7
+
+
+ 9540000-9699999
+ 5
+
+
+ 9700000-9849999
+ 7
+
+
+ 9850000-9999999
+ 5
+
+
+
+
+ 978-4
+ Japan
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9499999
+ 6
+
+
+ 9500000-9999999
+ 7
+
+
+
+
+ 978-5
+ former U.S.S.R
+
+
+ 0000000-0049999
+ 5
+
+
+ 0050000-0099999
+ 4
+
+
+ 0100000-1999999
+ 2
+
+
+ 2000000-3619999
+ 3
+
+
+ 3620000-3623999
+ 4
+
+
+ 3624000-3629999
+ 5
+
+
+ 3630000-4209999
+ 3
+
+
+ 4210000-4299999
+ 4
+
+
+ 4300000-4309999
+ 3
+
+
+ 4310000-4399999
+ 4
+
+
+ 4400000-4409999
+ 3
+
+
+ 4410000-4499999
+ 4
+
+
+ 4500000-6039999
+ 3
+
+
+ 6040000-6049999
+ 7
+
+
+ 6050000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9099999
+ 6
+
+
+ 9100000-9199999
+ 5
+
+
+ 9200000-9299999
+ 4
+
+
+ 9300000-9499999
+ 5
+
+
+ 9500000-9500999
+ 7
+
+
+ 9501000-9799999
+ 4
+
+
+ 9800000-9899999
+ 5
+
+
+ 9900000-9909999
+ 7
+
+
+ 9910000-9999999
+ 4
+
+
+
+
+ 978-600
+ Iran
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-4999999
+ 3
+
+
+ 5000000-8999999
+ 4
+
+
+ 9000000-9867999
+ 5
+
+
+ 9868000-9929999
+ 4
+
+
+ 9930000-9959999
+ 3
+
+
+ 9960000-9999999
+ 5
+
+
+
+
+ 978-601
+ Kazakhstan
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-7999999
+ 4
+
+
+ 8000000-8499999
+ 5
+
+
+ 8500000-9999999
+ 2
+
+
+
+
+ 978-602
+ Indonesia
+
+
+ 0000000-0699999
+ 2
+
+
+ 0700000-1399999
+ 4
+
+
+ 1400000-1499999
+ 5
+
+
+ 1500000-1699999
+ 4
+
+
+ 1700000-1999999
+ 5
+
+
+ 2000000-4999999
+ 3
+
+
+ 5000000-5399999
+ 5
+
+
+ 5400000-5999999
+ 4
+
+
+ 6000000-6199999
+ 5
+
+
+ 6200000-6999999
+ 4
+
+
+ 7000000-7499999
+ 5
+
+
+ 7500000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-603
+ Saudi Arabia
+
+
+ 0000000-0499999
+ 2
+
+
+ 0500000-4999999
+ 2
+
+
+ 5000000-7999999
+ 3
+
+
+ 8000000-8999999
+ 4
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-604
+ Vietnam
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-3999999
+ 3
+
+
+ 4000000-4699999
+ 2
+
+
+ 4700000-4979999
+ 3
+
+
+ 4980000-4999999
+ 4
+
+
+ 5000000-8999999
+ 2
+
+
+ 9000000-9799999
+ 3
+
+
+ 9800000-9999999
+ 4
+
+
+
+
+ 978-605
+ Turkey
+
+
+ 0000000-0299999
+ 2
+
+
+ 0300000-0399999
+ 3
+
+
+ 0400000-0599999
+ 2
+
+
+ 0600000-0699999
+ 5
+
+
+ 0700000-0999999
+ 2
+
+
+ 1000000-1999999
+ 3
+
+
+ 2000000-2399999
+ 4
+
+
+ 2400000-3999999
+ 3
+
+
+ 4000000-5999999
+ 4
+
+
+ 6000000-7499999
+ 5
+
+
+ 7500000-7999999
+ 4
+
+
+ 8000000-8999999
+ 5
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-606
+ Romania
+
+
+ 0000000-0999999
+ 3
+
+
+ 1000000-4999999
+ 2
+
+
+ 5000000-7999999
+ 3
+
+
+ 8000000-9099999
+ 4
+
+
+ 9100000-9199999
+ 3
+
+
+ 9200000-9599999
+ 5
+
+
+ 9600000-9749999
+ 4
+
+
+ 9750000-9999999
+ 3
+
+
+
+
+ 978-607
+ Mexico
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-5929999
+ 3
+
+
+ 5930000-5999999
+ 5
+
+
+ 6000000-7499999
+ 3
+
+
+ 7500000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-608
+ North Macedonia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-1999999
+ 2
+
+
+ 2000000-4499999
+ 3
+
+
+ 4500000-6499999
+ 4
+
+
+ 6500000-6999999
+ 5
+
+
+ 7000000-9999999
+ 1
+
+
+
+
+ 978-609
+ Lithuania
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-611
+ Thailand
+
+
+ 0000000-9999999
+ 0
+
+
+
+
+ 978-612
+ Peru
+
+
+ 0000000-2999999
+ 2
+
+
+ 3000000-3999999
+ 3
+
+
+ 4000000-4499999
+ 4
+
+
+ 4500000-4999999
+ 5
+
+
+ 5000000-5149999
+ 4
+
+
+ 5150000-9999999
+ 0
+
+
+
+
+ 978-613
+ Mauritius
+
+
+ 0000000-9999999
+ 1
+
+
+
+
+ 978-614
+ Lebanon
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-615
+ Hungary
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-4999999
+ 3
+
+
+ 5000000-7999999
+ 4
+
+
+ 8000000-8999999
+ 5
+
+
+ 9000000-9999999
+ 0
+
+
+
+
+ 978-616
+ Thailand
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8999999
+ 4
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-617
+ Ukraine
+
+
+ 0000000-4999999
+ 2
+
+
+ 5000000-6999999
+ 3
+
+
+ 7000000-8999999
+ 4
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-618
+ Greece
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-4999999
+ 3
+
+
+ 5000000-7999999
+ 4
+
+
+ 8000000-9999999
+ 5
+
+
+
+
+ 978-619
+ Bulgaria
+
+
+ 0000000-1499999
+ 2
+
+
+ 1500000-6999999
+ 3
+
+
+ 7000000-8999999
+ 4
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-620
+ Mauritius
+
+
+ 0000000-9999999
+ 1
+
+
+
+
+ 978-621
+ Philippines
+
+
+ 0000000-2999999
+ 2
+
+
+ 3000000-3999999
+ 0
+
+
+ 4000000-5999999
+ 3
+
+
+ 6000000-7999999
+ 0
+
+
+ 8000000-8999999
+ 4
+
+
+ 9000000-9499999
+ 0
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-622
+ Iran
+
+
+ 0000000-1099999
+ 2
+
+
+ 1100000-1999999
+ 0
+
+
+ 2000000-4249999
+ 3
+
+
+ 4250000-5199999
+ 0
+
+
+ 5200000-8499999
+ 4
+
+
+ 8500000-8999999
+ 0
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-623
+ Indonesia
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-1299999
+ 0
+
+
+ 1300000-4999999
+ 3
+
+
+ 5000000-5249999
+ 0
+
+
+ 5250000-8799999
+ 4
+
+
+ 8800000-9999999
+ 5
+
+
+
+
+ 978-624
+ Sri Lanka
+
+
+ 0000000-0499999
+ 2
+
+
+ 0500000-1999999
+ 0
+
+
+ 2000000-2499999
+ 3
+
+
+ 2500000-4999999
+ 0
+
+
+ 5000000-6449999
+ 4
+
+
+ 6450000-9449999
+ 0
+
+
+ 9450000-9999999
+ 5
+
+
+
+
+ 978-625
+ Turkey
+
+
+ 0000000-0099999
+ 2
+
+
+ 0100000-3649999
+ 0
+
+
+ 3650000-4429999
+ 3
+
+
+ 4430000-4449999
+ 5
+
+
+ 4450000-4499999
+ 3
+
+
+ 4500000-6349999
+ 0
+
+
+ 6350000-7793999
+ 4
+
+
+ 7794000-7794999
+ 5
+
+
+ 7795000-8499999
+ 4
+
+
+ 8500000-9899999
+ 0
+
+
+ 9900000-9999999
+ 5
+
+
+
+
+ 978-626
+ Taiwan
+
+
+ 0000000-0499999
+ 2
+
+
+ 0500000-2999999
+ 0
+
+
+ 3000000-4999999
+ 3
+
+
+ 5000000-6999999
+ 0
+
+
+ 7000000-7999999
+ 4
+
+
+ 8000000-9499999
+ 0
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-627
+ Pakistan
+
+
+ 0000000-2999999
+ 0
+
+
+ 3000000-3199999
+ 2
+
+
+ 3200000-4999999
+ 0
+
+
+ 5000000-5249999
+ 3
+
+
+ 5250000-7499999
+ 0
+
+
+ 7500000-7999999
+ 4
+
+
+ 8000000-9999999
+ 0
+
+
+
+
+ 978-628
+ Colombia
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-4999999
+ 0
+
+
+ 5000000-5499999
+ 3
+
+
+ 5500000-7499999
+ 0
+
+
+ 7500000-8499999
+ 4
+
+
+ 8500000-9499999
+ 0
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-629
+ Malaysia
+
+
+ 0000000-0299999
+ 2
+
+
+ 0300000-4699999
+ 0
+
+
+ 4700000-4999999
+ 3
+
+
+ 5000000-7499999
+ 0
+
+
+ 7500000-7999999
+ 4
+
+
+ 8000000-9649999
+ 0
+
+
+ 9650000-9999999
+ 5
+
+
+
+
+ 978-630
+ Romania
+
+
+ 0000000-2999999
+ 0
+
+
+ 3000000-3499999
+ 3
+
+
+ 3500000-6499999
+ 0
+
+
+ 6500000-6849999
+ 4
+
+
+ 6850000-9999999
+ 0
+
+
+
+
+ 978-631
+ Argentina
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-2999999
+ 0
+
+
+ 3000000-3999999
+ 3
+
+
+ 4000000-6499999
+ 0
+
+
+ 6500000-7499999
+ 4
+
+
+ 7500000-8999999
+ 0
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-65
+ Brazil
+
+
+ 0000000-0199999
+ 2
+
+
+ 0200000-2499999
+ 0
+
+
+ 2500000-2999999
+ 3
+
+
+ 3000000-3029999
+ 3
+
+
+ 3030000-4999999
+ 0
+
+
+ 5000000-5129999
+ 4
+
+
+ 5130000-5349999
+ 0
+
+
+ 5350000-6149999
+ 4
+
+
+ 6150000-7999999
+ 0
+
+
+ 8000000-8182499
+ 5
+
+
+ 8182500-8449999
+ 0
+
+
+ 8450000-8999999
+ 5
+
+
+ 9000000-9024499
+ 6
+
+
+ 9024500-9799999
+ 0
+
+
+ 9800000-9999999
+ 6
+
+
+
+
+ 978-7
+ China, People's Republic
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-4999999
+ 3
+
+
+ 5000000-7999999
+ 4
+
+
+ 8000000-8999999
+ 5
+
+
+ 9000000-9999999
+ 6
+
+
+
+
+ 978-80
+ former Czechoslovakia
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-5299999
+ 3
+
+
+ 5300000-5499999
+ 5
+
+
+ 5500000-6899999
+ 3
+
+
+ 6900000-6999999
+ 5
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9989999
+ 6
+
+
+ 9990000-9999999
+ 5
+
+
+
+
+ 978-81
+ India
+
+
+ 0000000-1899999
+ 2
+
+
+ 1900000-1999999
+ 5
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9999999
+ 6
+
+
+
+
+ 978-82
+ Norway
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6899999
+ 3
+
+
+ 6900000-6999999
+ 6
+
+
+ 7000000-8999999
+ 4
+
+
+ 9000000-9899999
+ 5
+
+
+ 9900000-9999999
+ 6
+
+
+
+
+ 978-83
+ Poland
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-5999999
+ 3
+
+
+ 6000000-6999999
+ 5
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9999999
+ 6
+
+
+
+
+ 978-84
+ Spain
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-1049999
+ 5
+
+
+ 1050000-1199999
+ 4
+
+
+ 1200000-1299999
+ 6
+
+
+ 1300000-1399999
+ 4
+
+
+ 1400000-1499999
+ 3
+
+
+ 1500000-1999999
+ 5
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9199999
+ 4
+
+
+ 9200000-9239999
+ 6
+
+
+ 9240000-9299999
+ 5
+
+
+ 9300000-9499999
+ 6
+
+
+ 9500000-9699999
+ 5
+
+
+ 9700000-9999999
+ 4
+
+
+
+
+ 978-85
+ Brazil
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-4549999
+ 3
+
+
+ 4550000-4552999
+ 6
+
+
+ 4553000-4559999
+ 5
+
+
+ 4560000-5289999
+ 3
+
+
+ 5290000-5319999
+ 5
+
+
+ 5320000-5339999
+ 4
+
+
+ 5340000-5399999
+ 3
+
+
+ 5400000-5402999
+ 5
+
+
+ 5403000-5403999
+ 5
+
+
+ 5404000-5404999
+ 6
+
+
+ 5405000-5408999
+ 5
+
+
+ 5409000-5409999
+ 6
+
+
+ 5410000-5439999
+ 5
+
+
+ 5440000-5479999
+ 4
+
+
+ 5480000-5499999
+ 5
+
+
+ 5500000-5999999
+ 4
+
+
+ 6000000-6999999
+ 5
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9249999
+ 6
+
+
+ 9250000-9449999
+ 5
+
+
+ 9450000-9599999
+ 4
+
+
+ 9600000-9799999
+ 2
+
+
+ 9800000-9999999
+ 5
+
+
+
+
+ 978-86
+ former Yugoslavia
+
+
+ 0000000-2999999
+ 2
+
+
+ 3000000-5999999
+ 3
+
+
+ 6000000-7999999
+ 4
+
+
+ 8000000-8999999
+ 5
+
+
+ 9000000-9999999
+ 6
+
+
+
+
+ 978-87
+ Denmark
+
+
+ 0000000-2999999
+ 2
+
+
+ 3000000-3999999
+ 0
+
+
+ 4000000-6499999
+ 3
+
+
+ 6500000-6999999
+ 0
+
+
+ 7000000-7999999
+ 4
+
+
+ 8000000-8499999
+ 0
+
+
+ 8500000-9499999
+ 5
+
+
+ 9500000-9699999
+ 0
+
+
+ 9700000-9999999
+ 6
+
+
+
+
+ 978-88
+ Italy
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-3119999
+ 3
+
+
+ 3120000-3149999
+ 5
+
+
+ 3150000-3189999
+ 3
+
+
+ 3190000-3229999
+ 5
+
+
+ 3230000-3269999
+ 3
+
+
+ 3270000-3389999
+ 4
+
+
+ 3390000-3609999
+ 3
+
+
+ 3610000-3629999
+ 4
+
+
+ 3630000-5489999
+ 3
+
+
+ 5490000-5549999
+ 4
+
+
+ 5550000-5999999
+ 3
+
+
+ 6000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9099999
+ 6
+
+
+ 9100000-9269999
+ 3
+
+
+ 9270000-9399999
+ 4
+
+
+ 9400000-9479999
+ 6
+
+
+ 9480000-9999999
+ 5
+
+
+
+
+ 978-89
+ Korea, Republic
+
+
+ 0000000-2499999
+ 2
+
+
+ 2500000-5499999
+ 3
+
+
+ 5500000-8499999
+ 4
+
+
+ 8500000-9499999
+ 5
+
+
+ 9500000-9699999
+ 6
+
+
+ 9700000-9899999
+ 5
+
+
+ 9900000-9999999
+ 3
+
+
+
+
+ 978-90
+ Netherlands
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-4999999
+ 3
+
+
+ 5000000-6999999
+ 4
+
+
+ 7000000-7999999
+ 5
+
+
+ 8000000-8499999
+ 6
+
+
+ 8500000-8999999
+ 4
+
+
+ 9000000-9099999
+ 2
+
+
+ 9100000-9399999
+ 0
+
+
+ 9400000-9499999
+ 2
+
+
+ 9500000-9999999
+ 0
+
+
+
+
+ 978-91
+ Sweden
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-4999999
+ 2
+
+
+ 5000000-6499999
+ 3
+
+
+ 6500000-6999999
+ 0
+
+
+ 7000000-8199999
+ 4
+
+
+ 8200000-8499999
+ 0
+
+
+ 8500000-9499999
+ 5
+
+
+ 9500000-9699999
+ 0
+
+
+ 9700000-9999999
+ 6
+
+
+
+
+ 978-92
+ International NGO Publishers and EU Organizations
+
+
+ 0000000-5999999
+ 1
+
+
+ 6000000-7999999
+ 2
+
+
+ 8000000-8999999
+ 3
+
+
+ 9000000-9499999
+ 4
+
+
+ 9500000-9899999
+ 5
+
+
+ 9900000-9999999
+ 6
+
+
+
+
+ 978-93
+ India
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-4999999
+ 3
+
+
+ 5000000-7999999
+ 4
+
+
+ 8000000-9599999
+ 5
+
+
+ 9600000-9999999
+ 6
+
+
+
+
+ 978-94
+ Netherlands
+
+
+ 0000000-5999999
+ 3
+
+
+ 6000000-8999999
+ 4
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-950
+ Argentina
+
+
+ 0000000-4999999
+ 2
+
+
+ 5000000-8999999
+ 3
+
+
+ 9000000-9899999
+ 4
+
+
+ 9900000-9999999
+ 5
+
+
+
+
+ 978-951
+ Finland
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-5499999
+ 2
+
+
+ 5500000-8899999
+ 3
+
+
+ 8900000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-952
+ Finland
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-4999999
+ 3
+
+
+ 5000000-5999999
+ 4
+
+
+ 6000000-6499999
+ 2
+
+
+ 6500000-6599999
+ 5
+
+
+ 6600000-6699999
+ 4
+
+
+ 6700000-6999999
+ 5
+
+
+ 7000000-7999999
+ 4
+
+
+ 8000000-9499999
+ 2
+
+
+ 9500000-9899999
+ 4
+
+
+ 9900000-9999999
+ 5
+
+
+
+
+ 978-953
+ Croatia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-1499999
+ 2
+
+
+ 1500000-4799999
+ 3
+
+
+ 4800000-4999999
+ 5
+
+
+ 5000000-5009999
+ 3
+
+
+ 5010000-5099999
+ 5
+
+
+ 5100000-5499999
+ 2
+
+
+ 5500000-5999999
+ 5
+
+
+ 6000000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-954
+ Bulgaria
+
+
+ 0000000-2899999
+ 2
+
+
+ 2900000-2999999
+ 4
+
+
+ 3000000-7999999
+ 3
+
+
+ 8000000-8999999
+ 4
+
+
+ 9000000-9299999
+ 5
+
+
+ 9300000-9999999
+ 4
+
+
+
+
+ 978-955
+ Sri Lanka
+
+
+ 0000000-1999999
+ 4
+
+
+ 2000000-3399999
+ 2
+
+
+ 3400000-3549999
+ 4
+
+
+ 3550000-3599999
+ 5
+
+
+ 3600000-3799999
+ 4
+
+
+ 3800000-3899999
+ 5
+
+
+ 3900000-4099999
+ 4
+
+
+ 4100000-4499999
+ 5
+
+
+ 4500000-4999999
+ 4
+
+
+ 5000000-5499999
+ 5
+
+
+ 5500000-7109999
+ 3
+
+
+ 7110000-7149999
+ 5
+
+
+ 7150000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-956
+ Chile
+
+
+ 0000000-0899999
+ 2
+
+
+ 0900000-0999999
+ 5
+
+
+ 1000000-1999999
+ 2
+
+
+ 2000000-5999999
+ 3
+
+
+ 6000000-6999999
+ 4
+
+
+ 7000000-9999999
+ 4
+
+
+
+
+ 978-957
+ Taiwan
+
+
+ 0000000-0299999
+ 2
+
+
+ 0300000-0499999
+ 4
+
+
+ 0500000-1999999
+ 2
+
+
+ 2000000-2099999
+ 4
+
+
+ 2100000-2799999
+ 2
+
+
+ 2800000-3099999
+ 5
+
+
+ 3100000-4399999
+ 2
+
+
+ 4400000-8199999
+ 3
+
+
+ 8200000-9699999
+ 4
+
+
+ 9700000-9999999
+ 5
+
+
+
+
+ 978-958
+ Colombia
+
+
+ 0000000-4999999
+ 2
+
+
+ 5000000-5099999
+ 3
+
+
+ 5100000-5199999
+ 4
+
+
+ 5200000-5399999
+ 5
+
+
+ 5400000-5599999
+ 4
+
+
+ 5600000-5999999
+ 5
+
+
+ 6000000-7999999
+ 3
+
+
+ 8000000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-959
+ Cuba
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-9999999
+ 5
+
+
+
+
+ 978-960
+ Greece
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6599999
+ 3
+
+
+ 6600000-6899999
+ 4
+
+
+ 6900000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-9299999
+ 5
+
+
+ 9300000-9399999
+ 2
+
+
+ 9400000-9799999
+ 4
+
+
+ 9800000-9999999
+ 5
+
+
+
+
+ 978-961
+ Slovenia
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-5999999
+ 3
+
+
+ 6000000-8999999
+ 4
+
+
+ 9000000-9799999
+ 5
+
+
+ 9800000-9999999
+ 0
+
+
+
+
+ 978-962
+ Hong Kong, China
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8699999
+ 5
+
+
+ 8700000-8999999
+ 4
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-963
+ Hungary
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-964
+ Iran
+
+
+ 0000000-1499999
+ 2
+
+
+ 1500000-2499999
+ 3
+
+
+ 2500000-2999999
+ 4
+
+
+ 3000000-5499999
+ 3
+
+
+ 5500000-8999999
+ 4
+
+
+ 9000000-9699999
+ 5
+
+
+ 9700000-9899999
+ 3
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-965
+ Israel
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-5999999
+ 3
+
+
+ 6000000-6999999
+ 0
+
+
+ 7000000-7999999
+ 4
+
+
+ 8000000-8999999
+ 0
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-966
+ Ukraine
+
+
+ 0000000-1299999
+ 2
+
+
+ 1300000-1399999
+ 3
+
+
+ 1400000-1499999
+ 2
+
+
+ 1500000-1699999
+ 4
+
+
+ 1700000-1999999
+ 3
+
+
+ 2000000-2789999
+ 4
+
+
+ 2790000-2899999
+ 3
+
+
+ 2900000-2999999
+ 4
+
+
+ 3000000-6999999
+ 3
+
+
+ 7000000-8999999
+ 4
+
+
+ 9000000-9099999
+ 5
+
+
+ 9100000-9499999
+ 3
+
+
+ 9500000-9799999
+ 5
+
+
+ 9800000-9999999
+ 3
+
+
+
+
+ 978-967
+ Malaysia
+
+
+ 0000000-0999999
+ 4
+
+
+ 1000000-1999999
+ 5
+
+
+ 2000000-2499999
+ 4
+
+
+ 2500000-2549999
+ 3
+
+
+ 2550000-2699999
+ 5
+
+
+ 2700000-2799999
+ 4
+
+
+ 2800000-2999999
+ 4
+
+
+ 3000000-4999999
+ 3
+
+
+ 5000000-5999999
+ 4
+
+
+ 6000000-8999999
+ 2
+
+
+ 9000000-9899999
+ 3
+
+
+ 9900000-9989999
+ 4
+
+
+ 9990000-9999999
+ 5
+
+
+
+
+ 978-968
+ Mexico
+
+
+ 0100000-3999999
+ 2
+
+
+ 4000000-4999999
+ 3
+
+
+ 5000000-7999999
+ 4
+
+
+ 8000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-969
+ Pakistan
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-2099999
+ 2
+
+
+ 2100000-2199999
+ 3
+
+
+ 2200000-2299999
+ 4
+
+
+ 2300000-2399999
+ 5
+
+
+ 2400000-3999999
+ 2
+
+
+ 4000000-7499999
+ 3
+
+
+ 7500000-9999999
+ 4
+
+
+
+
+ 978-970
+ Mexico
+
+
+ 0100000-5999999
+ 2
+
+
+ 6000000-8999999
+ 3
+
+
+ 9000000-9099999
+ 4
+
+
+ 9100000-9699999
+ 5
+
+
+ 9700000-9999999
+ 4
+
+
+
+
+ 978-971
+ Philippines
+
+
+ 0000000-0159999
+ 3
+
+
+ 0160000-0199999
+ 4
+
+
+ 0200000-0299999
+ 2
+
+
+ 0300000-0599999
+ 4
+
+
+ 0600000-4999999
+ 2
+
+
+ 5000000-8499999
+ 3
+
+
+ 8500000-9099999
+ 4
+
+
+ 9100000-9599999
+ 5
+
+
+ 9600000-9699999
+ 4
+
+
+ 9700000-9899999
+ 2
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-972
+ Portugal
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-5499999
+ 2
+
+
+ 5500000-7999999
+ 3
+
+
+ 8000000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-973
+ Romania
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-1699999
+ 3
+
+
+ 1700000-1999999
+ 4
+
+
+ 2000000-5499999
+ 2
+
+
+ 5500000-7599999
+ 3
+
+
+ 7600000-8499999
+ 4
+
+
+ 8500000-8899999
+ 5
+
+
+ 8900000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-974
+ Thailand
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8499999
+ 4
+
+
+ 8500000-8999999
+ 5
+
+
+ 9000000-9499999
+ 5
+
+
+ 9500000-9999999
+ 4
+
+
+
+
+ 978-975
+ Turkey
+
+
+ 0000000-0199999
+ 5
+
+
+ 0200000-2399999
+ 2
+
+
+ 2400000-2499999
+ 4
+
+
+ 2500000-5999999
+ 3
+
+
+ 6000000-9199999
+ 4
+
+
+ 9200000-9899999
+ 5
+
+
+ 9900000-9999999
+ 3
+
+
+
+
+ 978-976
+ Caribbean Community
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-5999999
+ 2
+
+
+ 6000000-7999999
+ 3
+
+
+ 8000000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-977
+ Egypt
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-4999999
+ 3
+
+
+ 5000000-6999999
+ 4
+
+
+ 7000000-8499999
+ 3
+
+
+ 8500000-8929999
+ 5
+
+
+ 8930000-8949999
+ 3
+
+
+ 8950000-8999999
+ 4
+
+
+ 9000000-9899999
+ 2
+
+
+ 9900000-9999999
+ 3
+
+
+
+
+ 978-978
+ Nigeria
+
+
+ 0000000-1999999
+ 3
+
+
+ 2000000-2999999
+ 4
+
+
+ 3000000-7799999
+ 5
+
+
+ 7800000-7999999
+ 3
+
+
+ 8000000-8999999
+ 4
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-979
+ Indonesia
+
+
+ 0000000-0999999
+ 3
+
+
+ 1000000-1499999
+ 4
+
+
+ 1500000-1999999
+ 5
+
+
+ 2000000-2999999
+ 2
+
+
+ 3000000-3999999
+ 4
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-980
+ Venezuela
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-5999999
+ 3
+
+
+ 6000000-9999999
+ 4
+
+
+
+
+ 978-981
+ Singapore
+
+
+ 0000000-1699999
+ 2
+
+
+ 1700000-1799999
+ 5
+
+
+ 1800000-1999999
+ 2
+
+
+ 2000000-2999999
+ 3
+
+
+ 3000000-3099999
+ 4
+
+
+ 3100000-3999999
+ 3
+
+
+ 4000000-9499999
+ 4
+
+
+ 9500000-9899999
+ 0
+
+
+ 9900000-9999999
+ 2
+
+
+
+
+ 978-982
+ South Pacific
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-6999999
+ 3
+
+
+ 7000000-8999999
+ 2
+
+
+ 9000000-9799999
+ 4
+
+
+ 9800000-9999999
+ 5
+
+
+
+
+ 978-983
+ Malaysia
+
+
+ 0000000-0199999
+ 2
+
+
+ 0200000-1999999
+ 3
+
+
+ 2000000-3999999
+ 4
+
+
+ 4000000-4499999
+ 5
+
+
+ 4500000-4999999
+ 2
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-8999999
+ 3
+
+
+ 9000000-9899999
+ 4
+
+
+ 9900000-9999999
+ 5
+
+
+
+
+ 978-984
+ Bangladesh
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-8999999
+ 4
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-985
+ Belarus
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-5999999
+ 3
+
+
+ 6000000-8799999
+ 4
+
+
+ 8800000-8999999
+ 3
+
+
+ 9000000-9999999
+ 5
+
+
+
+
+ 978-986
+ Taiwan
+
+
+ 0000000-0599999
+ 2
+
+
+ 0600000-0699999
+ 5
+
+
+ 0700000-0799999
+ 4
+
+
+ 0800000-1199999
+ 2
+
+
+ 1200000-5399999
+ 3
+
+
+ 5400000-7999999
+ 4
+
+
+ 8000000-9999999
+ 5
+
+
+
+
+ 978-987
+ Argentina
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-1999999
+ 4
+
+
+ 2000000-2999999
+ 5
+
+
+ 3000000-3599999
+ 2
+
+
+ 3600000-4199999
+ 4
+
+
+ 4200000-4399999
+ 2
+
+
+ 4400000-4499999
+ 4
+
+
+ 4500000-4899999
+ 5
+
+
+ 4900000-4999999
+ 4
+
+
+ 5000000-8249999
+ 3
+
+
+ 8250000-8279999
+ 4
+
+
+ 8280000-8299999
+ 5
+
+
+ 8300000-8499999
+ 4
+
+
+ 8500000-8899999
+ 2
+
+
+ 8900000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-988
+ Hong Kong, China
+
+
+ 0000000-1199999
+ 2
+
+
+ 1200000-1999999
+ 5
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-7999999
+ 5
+
+
+ 8000000-9699999
+ 4
+
+
+ 9700000-9999999
+ 5
+
+
+
+
+ 978-989
+ Portugal
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-3499999
+ 2
+
+
+ 3500000-3699999
+ 5
+
+
+ 3700000-5299999
+ 2
+
+
+ 5300000-5499999
+ 5
+
+
+ 5500000-7999999
+ 3
+
+
+ 8000000-9499999
+ 4
+
+
+ 9500000-9999999
+ 5
+
+
+
+
+ 978-9910
+ Uzbekistan
+
+
+ 0000000-7299999
+ 0
+
+
+ 7300000-7499999
+ 3
+
+
+ 7500000-9649999
+ 0
+
+
+ 9650000-9999999
+ 4
+
+
+
+
+ 978-9911
+ Montenegro
+
+
+ 0000000-1999999
+ 0
+
+
+ 2000000-2499999
+ 2
+
+
+ 2500000-5499999
+ 0
+
+
+ 5500000-7499999
+ 3
+
+
+ 7500000-9999999
+ 0
+
+
+
+
+ 978-9912
+ Tanzania
+
+
+ 0000000-3999999
+ 0
+
+
+ 4000000-4499999
+ 2
+
+
+ 4500000-7499999
+ 0
+
+
+ 7500000-7999999
+ 3
+
+
+ 8000000-9799999
+ 0
+
+
+ 9800000-9999999
+ 4
+
+
+
+
+ 978-9913
+ Uganda
+
+
+ 0000000-0799999
+ 2
+
+
+ 0800000-5999999
+ 0
+
+
+ 6000000-6999999
+ 3
+
+
+ 7000000-9549999
+ 0
+
+
+ 9550000-9999999
+ 4
+
+
+
+
+ 978-9914
+ Kenya
+
+
+ 0000000-3999999
+ 0
+
+
+ 4000000-5299999
+ 2
+
+
+ 5300000-6999999
+ 0
+
+
+ 7000000-7749999
+ 3
+
+
+ 7750000-9599999
+ 0
+
+
+ 9600000-9999999
+ 4
+
+
+
+
+ 978-9915
+ Uruguay
+
+
+ 0000000-3999999
+ 0
+
+
+ 4000000-5999999
+ 2
+
+
+ 6000000-6499999
+ 0
+
+
+ 6500000-7999999
+ 3
+
+
+ 8000000-9299999
+ 0
+
+
+ 9300000-9999999
+ 4
+
+
+
+
+ 978-9916
+ Estonia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-3999999
+ 2
+
+
+ 4000000-5999999
+ 1
+
+
+ 6000000-7999999
+ 3
+
+
+ 8000000-8499999
+ 2
+
+
+ 8500000-8999999
+ 3
+
+
+ 9000000-9249999
+ 0
+
+
+ 9250000-9999999
+ 4
+
+
+
+
+ 978-9917
+ Bolivia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-2999999
+ 0
+
+
+ 3000000-3499999
+ 2
+
+
+ 3500000-5999999
+ 0
+
+
+ 6000000-6999999
+ 3
+
+
+ 7000000-9799999
+ 0
+
+
+ 9800000-9999999
+ 4
+
+
+
+
+ 978-9918
+ Malta
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-1999999
+ 0
+
+
+ 2000000-2999999
+ 2
+
+
+ 3000000-5999999
+ 0
+
+
+ 6000000-7999999
+ 3
+
+
+ 8000000-9499999
+ 0
+
+
+ 9500000-9999999
+ 4
+
+
+
+
+ 978-9919
+ Mongolia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-1999999
+ 0
+
+
+ 2000000-2999999
+ 2
+
+
+ 3000000-4999999
+ 0
+
+
+ 5000000-5999999
+ 3
+
+
+ 6000000-8999999
+ 0
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9920
+ Morocco
+
+
+ 0000000-2999999
+ 0
+
+
+ 3000000-4299999
+ 2
+
+
+ 4300000-4999999
+ 0
+
+
+ 5000000-7999999
+ 3
+
+
+ 8000000-8749999
+ 0
+
+
+ 8750000-9999999
+ 4
+
+
+
+
+ 978-9921
+ Kuwait
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-2999999
+ 0
+
+
+ 3000000-3999999
+ 2
+
+
+ 4000000-6999999
+ 0
+
+
+ 7000000-8999999
+ 3
+
+
+ 9000000-9699999
+ 0
+
+
+ 9700000-9999999
+ 4
+
+
+
+
+ 978-9922
+ Iraq
+
+
+ 0000000-1999999
+ 0
+
+
+ 2000000-2999999
+ 2
+
+
+ 3000000-5999999
+ 0
+
+
+ 6000000-7999999
+ 3
+
+
+ 8000000-8499999
+ 0
+
+
+ 8500000-9999999
+ 4
+
+
+
+
+ 978-9923
+ Jordan
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-5999999
+ 2
+
+
+ 6000000-6999999
+ 0
+
+
+ 7000000-8999999
+ 3
+
+
+ 9000000-9399999
+ 0
+
+
+ 9400000-9999999
+ 4
+
+
+
+
+ 978-9924
+ Cambodia
+
+
+ 0000000-2999999
+ 0
+
+
+ 3000000-3999999
+ 2
+
+
+ 4000000-4999999
+ 0
+
+
+ 5000000-6499999
+ 3
+
+
+ 6500000-8999999
+ 0
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9925
+ Cyprus
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5499999
+ 2
+
+
+ 5500000-7349999
+ 3
+
+
+ 7350000-9999999
+ 4
+
+
+
+
+ 978-9926
+ Bosnia and Herzegovina
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-3999999
+ 2
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-9999999
+ 4
+
+
+
+
+ 978-9927
+ Qatar
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-3999999
+ 3
+
+
+ 4000000-4999999
+ 4
+
+
+ 5000000-9999999
+ 0
+
+
+
+
+ 978-9928
+ Albania
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-3999999
+ 3
+
+
+ 4000000-4999999
+ 4
+
+
+ 5000000-7999999
+ 0
+
+
+ 8000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 2
+
+
+
+
+ 978-9929
+ Guatemala
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-5499999
+ 2
+
+
+ 5500000-7999999
+ 3
+
+
+ 8000000-9999999
+ 4
+
+
+
+
+ 978-9930
+ Costa Rica
+
+
+ 0000000-4999999
+ 2
+
+
+ 5000000-9399999
+ 3
+
+
+ 9400000-9999999
+ 4
+
+
+
+
+ 978-9931
+ Algeria
+
+
+ 0000000-2399999
+ 2
+
+
+ 2400000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9932
+ Lao People's Democratic Republic
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-8499999
+ 3
+
+
+ 8500000-9999999
+ 4
+
+
+
+
+ 978-9933
+ Syria
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-3999999
+ 2
+
+
+ 4000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9934
+ Latvia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-4999999
+ 2
+
+
+ 5000000-7999999
+ 3
+
+
+ 8000000-9999999
+ 4
+
+
+
+
+ 978-9935
+ Iceland
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-3999999
+ 2
+
+
+ 4000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9936
+ Afghanistan
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-3999999
+ 2
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-9999999
+ 4
+
+
+
+
+ 978-9937
+ Nepal
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-4999999
+ 2
+
+
+ 5000000-7999999
+ 3
+
+
+ 8000000-9999999
+ 4
+
+
+
+
+ 978-9938
+ Tunisia
+
+
+ 0000000-7999999
+ 2
+
+
+ 8000000-9499999
+ 3
+
+
+ 9500000-9749999
+ 4
+
+
+ 9750000-9909999
+ 3
+
+
+ 9910000-9999999
+ 4
+
+
+
+
+ 978-9939
+ Armenia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-8999999
+ 3
+
+
+ 9000000-9599999
+ 4
+
+
+ 9600000-9799999
+ 3
+
+
+ 9800000-9999999
+ 2
+
+
+
+
+ 978-9940
+ Montenegro
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-4999999
+ 2
+
+
+ 5000000-8399999
+ 3
+
+
+ 8400000-8699999
+ 2
+
+
+ 8700000-9999999
+ 4
+
+
+
+
+ 978-9941
+ Georgia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-3999999
+ 2
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-8999999
+ 1
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9942
+ Ecuador
+
+
+ 0000000-5999999
+ 2
+
+
+ 6000000-6999999
+ 3
+
+
+ 7000000-7499999
+ 4
+
+
+ 7500000-8499999
+ 3
+
+
+ 8500000-8999999
+ 4
+
+
+ 9000000-9849999
+ 3
+
+
+ 9850000-9999999
+ 4
+
+
+
+
+ 978-9943
+ Uzbekistan
+
+
+ 0000000-2999999
+ 2
+
+
+ 3000000-3999999
+ 3
+
+
+ 4000000-9749999
+ 4
+
+
+ 9750000-9999999
+ 3
+
+
+
+
+ 978-9944
+ Turkey
+
+
+ 0000000-0999999
+ 4
+
+
+ 1000000-4999999
+ 3
+
+
+ 5000000-5999999
+ 4
+
+
+ 6000000-6999999
+ 2
+
+
+ 7000000-7999999
+ 3
+
+
+ 8000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-9945
+ Dominican Republic
+
+
+ 0000000-0099999
+ 2
+
+
+ 0100000-0799999
+ 3
+
+
+ 0800000-3999999
+ 2
+
+
+ 4000000-5699999
+ 3
+
+
+ 5700000-5799999
+ 2
+
+
+ 5800000-7999999
+ 3
+
+
+ 8000000-8099999
+ 2
+
+
+ 8100000-8499999
+ 3
+
+
+ 8500000-9999999
+ 4
+
+
+
+
+ 978-9946
+ Korea, P.D.R.
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-3999999
+ 2
+
+
+ 4000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9947
+ Algeria
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-9948
+ United Arab Emirates
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-8499999
+ 3
+
+
+ 8500000-9999999
+ 4
+
+
+
+
+ 978-9949
+ Estonia
+
+
+ 0000000-0899999
+ 2
+
+
+ 0900000-0999999
+ 3
+
+
+ 1000000-3999999
+ 2
+
+
+ 4000000-6999999
+ 3
+
+
+ 7000000-7199999
+ 2
+
+
+ 7200000-7499999
+ 4
+
+
+ 7500000-8999999
+ 2
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9950
+ Palestine
+
+
+ 0000000-2999999
+ 2
+
+
+ 3000000-8499999
+ 3
+
+
+ 8500000-9999999
+ 4
+
+
+
+
+ 978-9951
+ Kosova
+
+
+ 0000000-3899999
+ 2
+
+
+ 3900000-8499999
+ 3
+
+
+ 8500000-9799999
+ 4
+
+
+ 9800000-9999999
+ 3
+
+
+
+
+ 978-9952
+ Azerbaijan
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-3999999
+ 2
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-9999999
+ 4
+
+
+
+
+ 978-9953
+ Lebanon
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-3999999
+ 2
+
+
+ 4000000-5999999
+ 3
+
+
+ 6000000-8999999
+ 2
+
+
+ 9000000-9299999
+ 4
+
+
+ 9300000-9699999
+ 2
+
+
+ 9700000-9999999
+ 3
+
+
+
+
+ 978-9954
+ Morocco
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-3999999
+ 2
+
+
+ 4000000-7999999
+ 3
+
+
+ 8000000-9899999
+ 4
+
+
+ 9900000-9999999
+ 2
+
+
+
+
+ 978-9955
+ Lithuania
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-9299999
+ 3
+
+
+ 9300000-9999999
+ 4
+
+
+
+
+ 978-9956
+ Cameroon
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-3999999
+ 2
+
+
+ 4000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9957
+ Jordan
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-6499999
+ 3
+
+
+ 6500000-6799999
+ 2
+
+
+ 6800000-6999999
+ 3
+
+
+ 7000000-8499999
+ 2
+
+
+ 8500000-8799999
+ 4
+
+
+ 8800000-9999999
+ 2
+
+
+
+
+ 978-9958
+ Bosnia and Herzegovina
+
+
+ 0000000-0199999
+ 2
+
+
+ 0200000-0299999
+ 3
+
+
+ 0300000-0399999
+ 4
+
+
+ 0400000-0899999
+ 3
+
+
+ 0900000-0999999
+ 4
+
+
+ 1000000-1899999
+ 2
+
+
+ 1900000-1999999
+ 4
+
+
+ 2000000-4999999
+ 2
+
+
+ 5000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9959
+ Libya
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-9499999
+ 3
+
+
+ 9500000-9699999
+ 4
+
+
+ 9700000-9799999
+ 3
+
+
+ 9800000-9999999
+ 2
+
+
+
+
+ 978-9960
+ Saudi Arabia
+
+
+ 0000000-5999999
+ 2
+
+
+ 6000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9961
+ Algeria
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-6999999
+ 2
+
+
+ 7000000-9499999
+ 3
+
+
+ 9500000-9999999
+ 4
+
+
+
+
+ 978-9962
+ Panama
+
+
+ 0000000-5499999
+ 2
+
+
+ 5500000-5599999
+ 4
+
+
+ 5600000-5999999
+ 2
+
+
+ 6000000-8499999
+ 3
+
+
+ 8500000-9999999
+ 4
+
+
+
+
+ 978-9963
+ Cyprus
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-2499999
+ 4
+
+
+ 2500000-2799999
+ 3
+
+
+ 2800000-2999999
+ 4
+
+
+ 3000000-5499999
+ 2
+
+
+ 5500000-7349999
+ 3
+
+
+ 7350000-7499999
+ 4
+
+
+ 7500000-9999999
+ 4
+
+
+
+
+ 978-9964
+ Ghana
+
+
+ 0000000-6999999
+ 1
+
+
+ 7000000-9499999
+ 2
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-9965
+ Kazakhstan
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9966
+ Kenya
+
+
+ 0000000-1399999
+ 3
+
+
+ 1400000-1499999
+ 2
+
+
+ 1500000-1999999
+ 4
+
+
+ 2000000-6999999
+ 2
+
+
+ 7000000-7499999
+ 4
+
+
+ 7500000-8209999
+ 3
+
+
+ 8210000-8249999
+ 4
+
+
+ 8250000-8259999
+ 3
+
+
+ 8260000-8289999
+ 4
+
+
+ 8290000-9599999
+ 3
+
+
+ 9600000-9999999
+ 4
+
+
+
+
+ 978-9967
+ Kyrgyz Republic
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9968
+ Costa Rica
+
+
+ 0000000-4999999
+ 2
+
+
+ 5000000-9399999
+ 3
+
+
+ 9400000-9999999
+ 4
+
+
+
+
+ 978-9969
+ Algeria
+
+
+ 0000000-0699999
+ 2
+
+
+ 0700000-4999999
+ 0
+
+
+ 5000000-6499999
+ 3
+
+
+ 6500000-9699999
+ 0
+
+
+ 9700000-9999999
+ 4
+
+
+
+
+ 978-9970
+ Uganda
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9971
+ Singapore
+
+
+ 0000000-5999999
+ 1
+
+
+ 6000000-8999999
+ 2
+
+
+ 9000000-9899999
+ 3
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-9972
+ Peru
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-1999999
+ 1
+
+
+ 2000000-2499999
+ 3
+
+
+ 2500000-2999999
+ 4
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9973
+ Tunisia
+
+
+ 0000000-0599999
+ 2
+
+
+ 0600000-0899999
+ 3
+
+
+ 0900000-0999999
+ 4
+
+
+ 1000000-6999999
+ 2
+
+
+ 7000000-9699999
+ 3
+
+
+ 9700000-9999999
+ 4
+
+
+
+
+ 978-9974
+ Uruguay
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5499999
+ 2
+
+
+ 5500000-7499999
+ 3
+
+
+ 7500000-8799999
+ 4
+
+
+ 8800000-9099999
+ 3
+
+
+ 9100000-9499999
+ 2
+
+
+ 9500000-9999999
+ 2
+
+
+
+
+ 978-9975
+ Moldova
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-2999999
+ 3
+
+
+ 3000000-3999999
+ 4
+
+
+ 4000000-4499999
+ 4
+
+
+ 4500000-8999999
+ 2
+
+
+ 9000000-9499999
+ 3
+
+
+ 9500000-9999999
+ 4
+
+
+
+
+ 978-9976
+ Tanzania
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-5799999
+ 4
+
+
+ 5800000-5899999
+ 3
+
+
+ 5900000-8999999
+ 2
+
+
+ 9000000-9899999
+ 3
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-9977
+ Costa Rica
+
+
+ 0000000-8999999
+ 2
+
+
+ 9000000-9899999
+ 3
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-9978
+ Ecuador
+
+
+ 0000000-2999999
+ 2
+
+
+ 3000000-3999999
+ 3
+
+
+ 4000000-9499999
+ 2
+
+
+ 9500000-9899999
+ 3
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-9979
+ Iceland
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-6499999
+ 2
+
+
+ 6500000-6599999
+ 3
+
+
+ 6600000-7599999
+ 2
+
+
+ 7600000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9980
+ Papua New Guinea
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-8999999
+ 2
+
+
+ 9000000-9899999
+ 3
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-9981
+ Morocco
+
+
+ 0000000-0999999
+ 2
+
+
+ 1000000-1599999
+ 3
+
+
+ 1600000-1999999
+ 4
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-9499999
+ 3
+
+
+ 9500000-9999999
+ 4
+
+
+
+
+ 978-9982
+ Zambia
+
+
+ 0000000-7999999
+ 2
+
+
+ 8000000-9899999
+ 3
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-9983
+ Gambia
+
+
+ 0000000-7999999
+ 0
+
+
+ 8000000-9499999
+ 2
+
+
+ 9500000-9899999
+ 3
+
+
+ 9900000-9999999
+ 4
+
+
+
+
+ 978-9984
+ Latvia
+
+
+ 0000000-4999999
+ 2
+
+
+ 5000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9985
+ Estonia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 4
+
+
+
+
+ 978-9986
+ Lithuania
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-8999999
+ 3
+
+
+ 9000000-9399999
+ 4
+
+
+ 9400000-9699999
+ 3
+
+
+ 9700000-9999999
+ 2
+
+
+
+
+ 978-9987
+ Tanzania
+
+
+ 0000000-3999999
+ 2
+
+
+ 4000000-8799999
+ 3
+
+
+ 8800000-9999999
+ 4
+
+
+
+
+ 978-9988
+ Ghana
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-5499999
+ 2
+
+
+ 5500000-7499999
+ 3
+
+
+ 7500000-9999999
+ 4
+
+
+
+
+ 978-9989
+ North Macedonia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-1999999
+ 3
+
+
+ 2000000-2999999
+ 4
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-9499999
+ 3
+
+
+ 9500000-9999999
+ 4
+
+
+
+
+ 978-99901
+ Bahrain
+
+
+ 0000000-4999999
+ 2
+
+
+ 5000000-7999999
+ 3
+
+
+ 8000000-9999999
+ 2
+
+
+
+
+ 978-99902
+ Reserved Agency
+
+
+ 0000000-9999999
+ 0
+
+
+
+
+ 978-99903
+ Mauritius
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99904
+ Curaçao
+
+
+ 0000000-5999999
+ 1
+
+
+ 6000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99905
+ Bolivia
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99906
+ Kuwait
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-6999999
+ 3
+
+
+ 7000000-8999999
+ 2
+
+
+ 9000000-9499999
+ 2
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-99908
+ Malawi
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99909
+ Malta
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-9499999
+ 2
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-99910
+ Sierra Leone
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99911
+ Lesotho
+
+
+ 0000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99912
+ Botswana
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-5999999
+ 3
+
+
+ 6000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99913
+ Andorra
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-3599999
+ 2
+
+
+ 3600000-5999999
+ 0
+
+
+ 6000000-6049999
+ 3
+
+
+ 6050000-9999999
+ 0
+
+
+
+
+ 978-99914
+ International NGO Publishers
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-6999999
+ 2
+
+
+ 7000000-7999999
+ 1
+
+
+ 8000000-8699999
+ 2
+
+
+ 8700000-8799999
+ 3
+
+
+ 8800000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99915
+ Maldives
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99916
+ Namibia
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-6999999
+ 2
+
+
+ 7000000-9999999
+ 3
+
+
+
+
+ 978-99917
+ Brunei Darussalam
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-8899999
+ 2
+
+
+ 8900000-9999999
+ 3
+
+
+
+
+ 978-99918
+ Faroe Islands
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99919
+ Benin
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-3999999
+ 3
+
+
+ 4000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99920
+ Andorra
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99921
+ Qatar
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-6999999
+ 2
+
+
+ 7000000-7999999
+ 3
+
+
+ 8000000-8999999
+ 1
+
+
+ 9000000-9999999
+ 2
+
+
+
+
+ 978-99922
+ Guatemala
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-6999999
+ 2
+
+
+ 7000000-9999999
+ 3
+
+
+
+
+ 978-99923
+ El Salvador
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99924
+ Nicaragua
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99925
+ Paraguay
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-1999999
+ 2
+
+
+ 2000000-2999999
+ 3
+
+
+ 3000000-3999999
+ 1
+
+
+ 4000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99926
+ Honduras
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-5999999
+ 2
+
+
+ 6000000-8699999
+ 3
+
+
+ 8700000-8999999
+ 2
+
+
+ 9000000-9999999
+ 2
+
+
+
+
+ 978-99927
+ Albania
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99928
+ Georgia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99929
+ Mongolia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99930
+ Armenia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99931
+ Seychelles
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99932
+ Malta
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-5999999
+ 2
+
+
+ 6000000-6999999
+ 3
+
+
+ 7000000-7999999
+ 1
+
+
+ 8000000-9999999
+ 2
+
+
+
+
+ 978-99933
+ Nepal
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99934
+ Dominican Republic
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99935
+ Haiti
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-6999999
+ 3
+
+
+ 7000000-8999999
+ 1
+
+
+ 9000000-9999999
+ 2
+
+
+
+
+ 978-99936
+ Bhutan
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99937
+ Macau
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99938
+ Srpska, Republic of
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-5999999
+ 2
+
+
+ 6000000-8999999
+ 3
+
+
+ 9000000-9999999
+ 2
+
+
+
+
+ 978-99939
+ Guatemala
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99940
+ Georgia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-6999999
+ 2
+
+
+ 7000000-9999999
+ 3
+
+
+
+
+ 978-99941
+ Armenia
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99942
+ Sudan
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99943
+ Albania
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99944
+ Ethiopia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99945
+ Namibia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99946
+ Nepal
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99947
+ Tajikistan
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-6999999
+ 2
+
+
+ 7000000-9999999
+ 3
+
+
+
+
+ 978-99948
+ Eritrea
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99949
+ Mauritius
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-8999999
+ 1
+
+
+ 9000000-9899999
+ 3
+
+
+ 9900000-9999999
+ 2
+
+
+
+
+ 978-99950
+ Cambodia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99951
+ Reserved Agency
+
+
+ 0000000-9999999
+ 0
+
+
+
+
+ 978-99952
+ Mali
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99953
+ Paraguay
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-7999999
+ 2
+
+
+ 8000000-9399999
+ 3
+
+
+ 9400000-9999999
+ 2
+
+
+
+
+ 978-99954
+ Bolivia
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-6999999
+ 2
+
+
+ 7000000-8799999
+ 3
+
+
+ 8800000-9999999
+ 2
+
+
+
+
+ 978-99955
+ Srpska, Republic of
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-5999999
+ 2
+
+
+ 6000000-7999999
+ 3
+
+
+ 8000000-9999999
+ 2
+
+
+
+
+ 978-99956
+ Albania
+
+
+ 0000000-5999999
+ 2
+
+
+ 6000000-8599999
+ 3
+
+
+ 8600000-9999999
+ 2
+
+
+
+
+ 978-99957
+ Malta
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-9499999
+ 3
+
+
+ 9500000-9999999
+ 2
+
+
+
+
+ 978-99958
+ Bahrain
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-9399999
+ 2
+
+
+ 9400000-9499999
+ 3
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-99959
+ Luxembourg
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99960
+ Malawi
+
+
+ 0000000-0699999
+ 0
+
+
+ 0700000-0999999
+ 3
+
+
+ 1000000-9499999
+ 2
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-99961
+ El Salvador
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-3699999
+ 3
+
+
+ 3700000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99962
+ Mongolia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99963
+ Cambodia
+
+
+ 0000000-4999999
+ 2
+
+
+ 5000000-9199999
+ 3
+
+
+ 9200000-9999999
+ 2
+
+
+
+
+ 978-99964
+ Nicaragua
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99965
+ Macau
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-3599999
+ 3
+
+
+ 3600000-6299999
+ 2
+
+
+ 6300000-9999999
+ 3
+
+
+
+
+ 978-99966
+ Kuwait
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-6999999
+ 2
+
+
+ 7000000-7999999
+ 3
+
+
+ 8000000-9699999
+ 2
+
+
+ 9700000-9999999
+ 3
+
+
+
+
+ 978-99967
+ Paraguay
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-5999999
+ 2
+
+
+ 6000000-9999999
+ 3
+
+
+
+
+ 978-99968
+ Botswana
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-5999999
+ 3
+
+
+ 6000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99969
+ Oman
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-7999999
+ 2
+
+
+ 8000000-9499999
+ 3
+
+
+ 9500000-9999999
+ 2
+
+
+
+
+ 978-99970
+ Haiti
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99971
+ Myanmar
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-8499999
+ 2
+
+
+ 8500000-9999999
+ 3
+
+
+
+
+ 978-99972
+ Faroe Islands
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99973
+ Mongolia
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99974
+ Bolivia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-2599999
+ 2
+
+
+ 2600000-3999999
+ 3
+
+
+ 4000000-6399999
+ 2
+
+
+ 6400000-6499999
+ 3
+
+
+ 6500000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99975
+ Tajikistan
+
+
+ 0000000-2999999
+ 1
+
+
+ 3000000-3999999
+ 3
+
+
+ 4000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99976
+ Srpska, Republic of
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-1599999
+ 2
+
+
+ 1600000-1999999
+ 3
+
+
+ 2000000-5999999
+ 2
+
+
+ 6000000-8199999
+ 3
+
+
+ 8200000-8999999
+ 2
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99977
+ Rwanda
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-3999999
+ 0
+
+
+ 4000000-6999999
+ 2
+
+
+ 7000000-7999999
+ 3
+
+
+ 8000000-9749999
+ 0
+
+
+ 9750000-9999999
+ 3
+
+
+
+
+ 978-99978
+ Mongolia
+
+
+ 0000000-4999999
+ 1
+
+
+ 5000000-6999999
+ 2
+
+
+ 7000000-9999999
+ 3
+
+
+
+
+ 978-99979
+ Honduras
+
+
+ 0000000-3999999
+ 1
+
+
+ 4000000-7999999
+ 2
+
+
+ 8000000-9999999
+ 3
+
+
+
+
+ 978-99980
+ Bhutan
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-2999999
+ 0
+
+
+ 3000000-5999999
+ 2
+
+
+ 6000000-7499999
+ 0
+
+
+ 7500000-9999999
+ 3
+
+
+
+
+ 978-99981
+ Macau
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-2699999
+ 0
+
+
+ 2700000-7499999
+ 2
+
+
+ 7500000-9999999
+ 3
+
+
+
+
+ 978-99982
+ Benin
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-4999999
+ 0
+
+
+ 5000000-6899999
+ 2
+
+
+ 6900000-8999999
+ 0
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99983
+ El Salvador
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-4999999
+ 0
+
+
+ 5000000-6999999
+ 2
+
+
+ 7000000-9499999
+ 0
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-99984
+ Brunei Darussalam
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-4999999
+ 0
+
+
+ 5000000-6999999
+ 2
+
+
+ 7000000-9499999
+ 0
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-99985
+ Tajikistan
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-3499999
+ 0
+
+
+ 3500000-7999999
+ 2
+
+
+ 8000000-8499999
+ 0
+
+
+ 8500000-9999999
+ 3
+
+
+
+
+ 978-99986
+ Myanmar
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-4999999
+ 0
+
+
+ 5000000-6999999
+ 2
+
+
+ 7000000-9499999
+ 0
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-99987
+ Luxembourg
+
+
+ 0000000-6999999
+ 0
+
+
+ 7000000-9999999
+ 3
+
+
+
+
+ 978-99988
+ Sudan
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-4999999
+ 0
+
+
+ 5000000-5499999
+ 2
+
+
+ 5500000-7999999
+ 0
+
+
+ 8000000-8249999
+ 3
+
+
+ 8250000-9999999
+ 0
+
+
+
+
+ 978-99989
+ Paraguay
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-4999999
+ 0
+
+
+ 5000000-6499999
+ 2
+
+
+ 6500000-8999999
+ 0
+
+
+ 9000000-9999999
+ 3
+
+
+
+
+ 978-99990
+ Ethiopia
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-4999999
+ 0
+
+
+ 5000000-5499999
+ 2
+
+
+ 5500000-9749999
+ 0
+
+
+ 9750000-9999999
+ 3
+
+
+
+
+ 978-99992
+ Oman
+
+
+ 0000000-1999999
+ 1
+
+
+ 2000000-4999999
+ 0
+
+
+ 5000000-6499999
+ 2
+
+
+ 6500000-9499999
+ 0
+
+
+ 9500000-9999999
+ 3
+
+
+
+
+ 978-99993
+ Mauritius
+
+
+ 0000000-0999999
+ 1
+
+
+ 1000000-4999999
+ 0
+
+
+ 5000000-5499999
+ 2
+
+
+ 5500000-9799999
+ 0
+
+
+ 9800000-9999999
+ 3
+
+
+
+
+ 979-10
+ France
+
+
+ 0000000-1999999
+ 2
+
+
+ 2000000-6999999
+ 3
+
+
+ 7000000-8999999
+ 4
+
+
+ 9000000-9759999
+ 5
+
+
+ 9760000-9999999
+ 6
+
+
+
+
+ 979-11
+ Korea, Republic
+
+
+ 0000000-2499999
+ 2
+
+
+ 2500000-5499999
+ 3
+
+
+ 5500000-8499999
+ 4
+
+
+ 8500000-9499999
+ 5
+
+
+ 9500000-9999999
+ 6
+
+
+
+
+ 979-12
+ Italy
+
+
+ 0000000-1999999
+ 0
+
+
+ 2000000-2999999
+ 3
+
+
+ 3000000-5449999
+ 0
+
+
+ 5450000-5999999
+ 4
+
+
+ 6000000-7999999
+ 0
+
+
+ 8000000-8499999
+ 5
+
+
+ 8500000-9999999
+ 0
+
+
+
+
+ 979-8
+ United States
+
+
+ 0000000-1999999
+ 0
+
+
+ 2000000-2299999
+ 3
+
+
+ 2300000-3499999
+ 0
+
+
+ 3500000-3999999
+ 4
+
+
+ 4000000-8499999
+ 4
+
+
+ 8500000-8849999
+ 4
+
+
+ 8850000-8999999
+ 5
+
+
+ 9000000-9849999
+ 0
+
+
+ 9850000-9899999
+ 7
+
+
+ 9900000-9999999
+ 0
+
+
+
+
+
diff --git a/bookwyrm/isbn/__init__.py b/bookwyrm/isbn/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/bookwyrm/isbn/isbn.py b/bookwyrm/isbn/isbn.py
new file mode 100644
index 000000000..4cc7f47dd
--- /dev/null
+++ b/bookwyrm/isbn/isbn.py
@@ -0,0 +1,123 @@
+""" Use the range message from isbn-international to hyphenate ISBNs """
+import os
+from typing import Optional
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+
+import requests
+
+from bookwyrm import settings
+
+
+def _get_rules(element: Element) -> list[Element]:
+ if (rules_el := element.find("Rules")) is not None:
+ return rules_el.findall("Rule")
+ return []
+
+
+class IsbnHyphenator:
+ """Class to manage the range message xml file and use it to hyphenate ISBNs"""
+
+ __range_message_url = "https://www.isbn-international.org/export_rangemessage.xml"
+ __range_file_path = os.path.join(
+ settings.BASE_DIR, "bookwyrm", "isbn", "RangeMessage.xml"
+ )
+ __element_tree = None
+
+ def update_range_message(self) -> None:
+ """Download the range message xml file and save it locally"""
+ response = requests.get(self.__range_message_url)
+ with open(self.__range_file_path, "w", encoding="utf-8") as file:
+ file.write(response.text)
+ self.__element_tree = None
+
+ def hyphenate(self, isbn_13: Optional[str]) -> Optional[str]:
+ """hyphenate the given ISBN-13 number using the range message"""
+ if isbn_13 is None:
+ return None
+
+ if self.__element_tree is None:
+ self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+ gs1_prefix = isbn_13[:3]
+ reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
+ if reg_group is None:
+ return isbn_13 # failed to hyphenate
+
+ registrant = self.__find_registrant(isbn_13, gs1_prefix, reg_group)
+ if registrant is None:
+ return isbn_13 # failed to hyphenate
+
+ publication = isbn_13[len(gs1_prefix) + len(reg_group) + len(registrant) : -1]
+ check_digit = isbn_13[-1:]
+ return "-".join((gs1_prefix, reg_group, registrant, publication, check_digit))
+
+ def __find_reg_group(self, isbn_13: str, gs1_prefix: str) -> Optional[str]:
+ if self.__element_tree is None:
+ self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+ ucc_prefixes_el = self.__element_tree.find("EAN.UCCPrefixes")
+ if ucc_prefixes_el is None:
+ return None
+
+ for ean_ucc_el in ucc_prefixes_el.findall("EAN.UCC"):
+ if (
+ prefix_el := ean_ucc_el.find("Prefix")
+ ) is not None and prefix_el.text == gs1_prefix:
+ for rule_el in _get_rules(ean_ucc_el):
+ length_el = rule_el.find("Length")
+ if length_el is None:
+ continue
+ length = int(text) if (text := length_el.text) else 0
+ if length == 0:
+ continue
+
+ range_el = rule_el.find("Range")
+ if range_el is None or range_el.text is None:
+ continue
+
+ reg_grp_range = [int(x[:length]) for x in range_el.text.split("-")]
+ reg_group = isbn_13[len(gs1_prefix) : len(gs1_prefix) + length]
+ if reg_grp_range[0] <= int(reg_group) <= reg_grp_range[1]:
+ return reg_group
+ return None
+ return None
+
+ def __find_registrant(
+ self, isbn_13: str, gs1_prefix: str, reg_group: str
+ ) -> Optional[str]:
+ from_ind = len(gs1_prefix) + len(reg_group)
+
+ if self.__element_tree is None:
+ self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+ reg_groups_el = self.__element_tree.find("RegistrationGroups")
+ if reg_groups_el is None:
+ return None
+
+ for group_el in reg_groups_el.findall("Group"):
+ if (
+ prefix_el := group_el.find("Prefix")
+ ) is not None and prefix_el.text == "-".join((gs1_prefix, reg_group)):
+ for rule_el in _get_rules(group_el):
+ length_el = rule_el.find("Length")
+ if length_el is None:
+ continue
+ length = int(text) if (text := length_el.text) else 0
+ if length == 0:
+ continue
+
+ range_el = rule_el.find("Range")
+ if range_el is None or range_el.text is None:
+ continue
+ registrant_range = [
+ int(x[:length]) for x in range_el.text.split("-")
+ ]
+ registrant = isbn_13[from_ind : from_ind + length]
+ if registrant_range[0] <= int(registrant) <= registrant_range[1]:
+ return registrant
+ return None
+ return None
+
+
+hyphenator_singleton = IsbnHyphenator()
diff --git a/bookwyrm/lists_stream.py b/bookwyrm/lists_stream.py
index 0977ad8c2..148b81a78 100644
--- a/bookwyrm/lists_stream.py
+++ b/bookwyrm/lists_stream.py
@@ -5,7 +5,7 @@ from django.db.models import signals, Count, Q
from bookwyrm import models
from bookwyrm.redis_store import RedisStore
-from bookwyrm.tasks import app, MEDIUM, HIGH
+from bookwyrm.tasks import app, LISTS
class ListsStream(RedisStore):
@@ -24,8 +24,7 @@ class ListsStream(RedisStore):
def add_list(self, book_list):
"""add a list to users' feeds"""
- # the pipeline contains all the add-to-stream activities
- self.add_object_to_related_stores(book_list)
+ self.add_object_to_stores(book_list, self.get_stores_for_object(book_list))
def add_user_lists(self, viewer, user):
"""add a user's lists to another user's feed"""
@@ -86,18 +85,19 @@ class ListsStream(RedisStore):
if group:
audience = audience.filter(
Q(id=book_list.user.id) # if the user is the list's owner
- | Q(following=book_list.user) # if the user is following the pwmer
+ | Q(following=book_list.user) # if the user is following the owner
# if a user is in the group
| Q(memberships__group__id=book_list.group.id)
)
else:
audience = audience.filter(
Q(id=book_list.user.id) # if the user is the list's owner
- | Q(following=book_list.user) # if the user is following the pwmer
+ | Q(following=book_list.user) # if the user is following the owner
)
return audience.distinct()
def get_stores_for_object(self, obj):
+ """the stores that an object belongs in"""
return [self.stream_id(u) for u in self.get_audience(obj)]
def get_lists_for_user(self, user): # pylint: disable=no-self-use
@@ -217,14 +217,14 @@ def add_list_on_account_create_command(user_id):
# ---- TASKS
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
def populate_lists_task(user_id):
"""background task for populating an empty list stream"""
user = models.User.objects.get(id=user_id)
ListsStream().populate_lists(user)
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
def remove_list_task(list_id, re_add=False):
"""remove a list from any stream it might be in"""
stores = models.User.objects.filter(local=True, is_active=True).values_list(
@@ -233,20 +233,20 @@ def remove_list_task(list_id, re_add=False):
# delete for every store
stores = [ListsStream().stream_id(idx) for idx in stores]
- ListsStream().remove_object_from_related_stores(list_id, stores=stores)
+ ListsStream().remove_object_from_stores(list_id, stores)
if re_add:
add_list_task.delay(list_id)
-@app.task(queue=HIGH)
+@app.task(queue=LISTS)
def add_list_task(list_id):
"""add a list to any stream it should be in"""
book_list = models.List.objects.get(id=list_id)
ListsStream().add_list(book_list)
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
"""remove all lists by a user from a viewer's stream"""
viewer = models.User.objects.get(id=viewer_id)
@@ -254,7 +254,7 @@ def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy)
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
def add_user_lists_task(viewer_id, user_id):
"""add all lists by a user to a viewer's stream"""
viewer = models.User.objects.get(id=viewer_id)
diff --git a/bookwyrm/management/commands/compile_themes.py b/bookwyrm/management/commands/compile_themes.py
new file mode 100644
index 000000000..95c6699ba
--- /dev/null
+++ b/bookwyrm/management/commands/compile_themes.py
@@ -0,0 +1,48 @@
+""" Our own command to all scss themes """
+import glob
+import os
+
+import sass
+
+from django.core.management.base import BaseCommand
+
+from sass_processor.apps import APPS_INCLUDE_DIRS
+from sass_processor.processor import SassProcessor
+from sass_processor.utils import get_custom_functions
+
+from bookwyrm import settings
+
+
+class Command(BaseCommand):
+ """command-line options"""
+
+ help = "SCSS compile all BookWyrm themes"
+
+ # pylint: disable=unused-argument
+ def handle(self, *args, **options):
+ """compile"""
+ themes_dir = os.path.join(
+ settings.BASE_DIR, "bookwyrm", "static", "css", "themes", "*.scss"
+ )
+ for theme_scss in glob.glob(themes_dir):
+ basename, _ = os.path.splitext(theme_scss)
+ theme_css = f"{basename}.css"
+ self.compile_sass(theme_scss, theme_css)
+
+ def compile_sass(self, sass_path, css_path):
+ compile_kwargs = {
+ "filename": sass_path,
+ "include_paths": SassProcessor.include_paths + APPS_INCLUDE_DIRS,
+ "custom_functions": get_custom_functions(),
+ "precision": getattr(settings, "SASS_PRECISION", 8),
+ "output_style": getattr(
+ settings,
+ "SASS_OUTPUT_STYLE",
+ "nested" if settings.DEBUG else "compressed",
+ ),
+ }
+
+ content = sass.compile(**compile_kwargs)
+ with open(css_path, "w") as f:
+ f.write(content)
+ self.stdout.write("Compiled SASS/SCSS file: '{0}'\n".format(sass_path))
diff --git a/bookwyrm/management/commands/confirm_email.py b/bookwyrm/management/commands/confirm_email.py
new file mode 100644
index 000000000..450da7eec
--- /dev/null
+++ b/bookwyrm/management/commands/confirm_email.py
@@ -0,0 +1,19 @@
+""" manually confirm e-mail of user """
+from django.core.management.base import BaseCommand
+
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+ """command-line options"""
+
+ help = "Manually confirm email for user"
+
+ def add_arguments(self, parser):
+ parser.add_argument("username")
+
+ def handle(self, *args, **options):
+ name = options["username"]
+ user = models.User.objects.get(localname=name)
+ user.reactivate()
+ self.stdout.write(self.style.SUCCESS("User's email is now confirmed."))
diff --git a/bookwyrm/management/commands/deduplicate_book_data.py b/bookwyrm/management/commands/deduplicate_book_data.py
index ed01a7843..dde7d133c 100644
--- a/bookwyrm/management/commands/deduplicate_book_data.py
+++ b/bookwyrm/management/commands/deduplicate_book_data.py
@@ -3,38 +3,7 @@ merge book data objects """
from django.core.management.base import BaseCommand
from django.db.models import Count
from bookwyrm import models
-
-
-def update_related(canonical, obj):
- """update all the models with fk to the object being removed"""
- # move related models to canonical
- related_models = [
- (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
- ]
- for (related_field, related_model) in related_models:
- related_objs = related_model.objects.filter(**{related_field: obj})
- for related_obj in related_objs:
- print("replacing in", related_model.__name__, related_field, related_obj.id)
- try:
- setattr(related_obj, related_field, canonical)
- related_obj.save()
- except TypeError:
- getattr(related_obj, related_field).add(canonical)
- getattr(related_obj, related_field).remove(obj)
-
-
-def copy_data(canonical, obj):
- """try to get the most data possible"""
- for data_field in obj._meta.get_fields():
- if not hasattr(data_field, "activitypub_field"):
- continue
- data_value = getattr(obj, data_field.name)
- if not data_value:
- continue
- if not getattr(canonical, data_field.name):
- print("setting data field", data_field.name, data_value)
- setattr(canonical, data_field.name, data_value)
- canonical.save()
+from bookwyrm.management.merge import merge_objects
def dedupe_model(model):
@@ -61,19 +30,16 @@ def dedupe_model(model):
print("keeping", canonical.remote_id)
for obj in objs[1:]:
print(obj.remote_id)
- copy_data(canonical, obj)
- update_related(canonical, obj)
- # remove the outdated entry
- obj.delete()
+ merge_objects(canonical, obj)
class Command(BaseCommand):
- """dedplucate allllll the book data models"""
+ """deduplicate allllll the book data models"""
help = "merges duplicate book data"
# pylint: disable=no-self-use,unused-argument
def handle(self, *args, **options):
- """run deudplications"""
+ """run deduplications"""
dedupe_model(models.Edition)
dedupe_model(models.Work)
dedupe_model(models.Author)
diff --git a/bookwyrm/management/commands/erase_streams.py b/bookwyrm/management/commands/erase_streams.py
index 9d971d699..ecd36006c 100644
--- a/bookwyrm/management/commands/erase_streams.py
+++ b/bookwyrm/management/commands/erase_streams.py
@@ -4,12 +4,7 @@ import redis
from bookwyrm import settings
-r = redis.Redis(
- host=settings.REDIS_ACTIVITY_HOST,
- port=settings.REDIS_ACTIVITY_PORT,
- password=settings.REDIS_ACTIVITY_PASSWORD,
- db=settings.REDIS_ACTIVITY_DB_INDEX,
-)
+r = redis.from_url(settings.REDIS_ACTIVITY_URL)
def erase_streams():
diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py
index 23020a0a6..ef8aff0fb 100644
--- a/bookwyrm/management/commands/initdb.py
+++ b/bookwyrm/management/commands/initdb.py
@@ -8,54 +8,64 @@ from bookwyrm import models
def init_groups():
"""permission levels"""
- groups = ["admin", "moderator", "editor"]
+ groups = ["admin", "owner", "moderator", "editor"]
for group in groups:
- Group.objects.create(name=group)
+ Group.objects.get_or_create(name=group)
def init_permissions():
"""permission types"""
permissions = [
+ {
+ "codename": "manage_registration",
+ "name": "allow or prevent user registration",
+ "groups": ["admin"],
+ },
+ {
+ "codename": "system_administration",
+ "name": "technical controls",
+ "groups": ["admin"],
+ },
{
"codename": "edit_instance_settings",
"name": "change the instance info",
- "groups": ["admin"],
+ "groups": ["admin", "owner"],
},
{
"codename": "set_user_group",
"name": "change what group a user is in",
- "groups": ["admin", "moderator"],
+ "groups": ["admin", "owner", "moderator"],
},
{
"codename": "control_federation",
"name": "control who to federate with",
- "groups": ["admin", "moderator"],
+ "groups": ["admin", "owner", "moderator"],
},
{
"codename": "create_invites",
"name": "issue invitations to join",
- "groups": ["admin", "moderator"],
+ "groups": ["admin", "owner", "moderator"],
},
{
"codename": "moderate_user",
"name": "deactivate or silence a user",
- "groups": ["admin", "moderator"],
+ "groups": ["admin", "owner", "moderator"],
},
{
"codename": "moderate_post",
"name": "delete other users' posts",
- "groups": ["admin", "moderator"],
+ "groups": ["admin", "owner", "moderator"],
},
{
"codename": "edit_book",
"name": "edit book info",
- "groups": ["admin", "moderator", "editor"],
+ "groups": ["admin", "owner", "moderator", "editor"],
},
]
content_type = ContentType.objects.get_for_model(models.User)
for permission in permissions:
- permission_obj = Permission.objects.create(
+ permission_obj, _ = Permission.objects.get_or_create(
codename=permission["codename"],
name=permission["name"],
content_type=content_type,
@@ -107,10 +117,12 @@ def init_connectors():
def init_settings():
"""info about the instance"""
+ group_editor = Group.objects.filter(name="editor").first()
models.SiteSettings.objects.create(
support_link="https://www.patreon.com/bookwyrm",
support_title="Patreon",
install_mode=True,
+ default_user_auth_group=group_editor,
)
diff --git a/bookwyrm/management/commands/merge_authors.py b/bookwyrm/management/commands/merge_authors.py
new file mode 100644
index 000000000..7465df147
--- /dev/null
+++ b/bookwyrm/management/commands/merge_authors.py
@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge author data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+ """merges two authors by ID"""
+
+ help = "merges specified authors into one"
+
+ MODEL = models.Author
diff --git a/bookwyrm/management/commands/merge_editions.py b/bookwyrm/management/commands/merge_editions.py
new file mode 100644
index 000000000..9ed696201
--- /dev/null
+++ b/bookwyrm/management/commands/merge_editions.py
@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge edition data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+ """merges two editions by ID"""
+
+ help = "merges specified editions into one"
+
+ MODEL = models.Edition
diff --git a/bookwyrm/management/commands/merge_works.py b/bookwyrm/management/commands/merge_works.py
new file mode 100644
index 000000000..619d0509a
--- /dev/null
+++ b/bookwyrm/management/commands/merge_works.py
@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge work data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+ """merges two works by ID"""
+
+ help = "merges specified works into one"
+
+ MODEL = models.Work
diff --git a/bookwyrm/management/commands/remove_2fa.py b/bookwyrm/management/commands/remove_2fa.py
new file mode 100644
index 000000000..1c9d5f71a
--- /dev/null
+++ b/bookwyrm/management/commands/remove_2fa.py
@@ -0,0 +1,22 @@
+"""deactivate two factor auth"""
+
+from django.core.management.base import BaseCommand, CommandError
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+ """command-line options"""
+
+ help = "Remove Two Factor Authorisation from user"
+
+ def add_arguments(self, parser):
+ parser.add_argument("username")
+
+ def handle(self, *args, **options):
+ name = options["username"]
+ user = models.User.objects.get(localname=name)
+ user.two_factor_auth = False
+ user.save(broadcast=False, update_fields=["two_factor_auth"])
+ self.stdout.write(
+ self.style.SUCCESS("Two Factor Authorisation was removed from user")
+ )
diff --git a/bookwyrm/management/commands/remove_editions.py b/bookwyrm/management/commands/remove_editions.py
index 9eb9b7da8..5cb430a93 100644
--- a/bookwyrm/management/commands/remove_editions.py
+++ b/bookwyrm/management/commands/remove_editions.py
@@ -33,10 +33,10 @@ def remove_editions():
class Command(BaseCommand):
- """dedplucate allllll the book data models"""
+ """deduplicate allllll the book data models"""
help = "merges duplicate book data"
# pylint: disable=no-self-use,unused-argument
def handle(self, *args, **options):
- """run deudplications"""
+ """run deduplications"""
remove_editions()
diff --git a/bookwyrm/management/commands/remove_remote_user_preview_images.py b/bookwyrm/management/commands/remove_remote_user_preview_images.py
new file mode 100644
index 000000000..d4dc131d8
--- /dev/null
+++ b/bookwyrm/management/commands/remove_remote_user_preview_images.py
@@ -0,0 +1,40 @@
+""" Remove preview images for remote users """
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+
+from bookwyrm import models, preview_images
+
+
+# pylint: disable=line-too-long
+class Command(BaseCommand):
+ """Remove preview images for remote users"""
+
+ help = "Remove preview images for remote users"
+
+ # pylint: disable=no-self-use,unused-argument
+ def handle(self, *args, **options):
+ """generate preview images"""
+ self.stdout.write(
+ " | Hello! I will be removing preview images from remote users."
+ )
+ self.stdout.write(
+ "🧑🚒 ⎨ This might take quite long if your instance has a lot of remote users."
+ )
+ self.stdout.write(" | ✧ Thank you for your patience ✧")
+
+ users = models.User.objects.filter(local=False).exclude(
+ Q(preview_image="") | Q(preview_image=None)
+ )
+
+ if len(users) > 0:
+ self.stdout.write(
+ f" → Remote user preview images ({len(users)}): ", ending=""
+ )
+ for user in users:
+ preview_images.remove_user_preview_image_task.delay(user.id)
+ self.stdout.write(".", ending="")
+ self.stdout.write(" OK 🖼")
+ else:
+ self.stdout.write(f" | There was no remote users with preview images.")
+
+ self.stdout.write("🧑🚒 ⎨ I’m all done! ✧ Enjoy ✧")
diff --git a/bookwyrm/management/commands/repair_editions.py b/bookwyrm/management/commands/repair_editions.py
new file mode 100644
index 000000000..304cd5e51
--- /dev/null
+++ b/bookwyrm/management/commands/repair_editions.py
@@ -0,0 +1,21 @@
+""" Repair editions with missing works """
+from django.core.management.base import BaseCommand
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+ """command-line options"""
+
+ help = "Repairs an edition that is in a broken state"
+
+ # pylint: disable=unused-argument
+ def handle(self, *args, **options):
+ """Find and repair broken editions"""
+ # Find broken editions
+ editions = models.Edition.objects.filter(parent_work__isnull=True)
+ self.stdout.write(f"Repairing {editions.count()} edition(s):")
+
+ # Do repair
+ for edition in editions:
+ edition.repair()
+ self.stdout.write(".", ending="")
diff --git a/bookwyrm/management/commands/revoke_preview_image_tasks.py b/bookwyrm/management/commands/revoke_preview_image_tasks.py
new file mode 100644
index 000000000..7b0947b12
--- /dev/null
+++ b/bookwyrm/management/commands/revoke_preview_image_tasks.py
@@ -0,0 +1,31 @@
+""" Actually let's not generate those preview images """
+import json
+from django.core.management.base import BaseCommand
+from bookwyrm.tasks import app
+
+
+class Command(BaseCommand):
+ """Find and revoke image tasks"""
+
+ # pylint: disable=unused-argument
+ def handle(self, *args, **options):
+ """revoke nonessential low priority tasks"""
+ types = [
+ "bookwyrm.preview_images.generate_edition_preview_image_task",
+ "bookwyrm.preview_images.generate_user_preview_image_task",
+ ]
+ self.stdout.write(" | Finding tasks of types:")
+ self.stdout.write("\n".join(types))
+ with app.pool.acquire(block=True) as conn:
+ tasks = conn.default_channel.client.lrange("low_priority", 0, -1)
+ self.stdout.write(f" | Found {len(tasks)} task(s) in low priority queue")
+
+ revoke_ids = []
+ for task in tasks:
+ task_json = json.loads(task)
+ task_type = task_json.get("headers", {}).get("task")
+ if task_type in types:
+ revoke_ids.append(task_json.get("headers", {}).get("id"))
+ self.stdout.write(".", ending="")
+ self.stdout.write(f"\n | Revoking {len(revoke_ids)} task(s)")
+ app.control.revoke(revoke_ids)
diff --git a/bookwyrm/management/merge.py b/bookwyrm/management/merge.py
new file mode 100644
index 000000000..f55229f18
--- /dev/null
+++ b/bookwyrm/management/merge.py
@@ -0,0 +1,50 @@
+from django.db.models import ManyToManyField
+
+
+def update_related(canonical, obj):
+ """update all the models with fk to the object being removed"""
+ # move related models to canonical
+ related_models = [
+ (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
+ ]
+ for (related_field, related_model) in related_models:
+ # Skip the ManyToMany fields that aren’t auto-created. These
+ # should have a corresponding OneToMany field in the model for
+ # the linking table anyway. If we update it through that model
+ # instead then we won’t lose the extra fields in the linking
+ # table.
+ related_field_obj = related_model._meta.get_field(related_field)
+ if isinstance(related_field_obj, ManyToManyField):
+ through = related_field_obj.remote_field.through
+ if not through._meta.auto_created:
+ continue
+ related_objs = related_model.objects.filter(**{related_field: obj})
+ for related_obj in related_objs:
+ print("replacing in", related_model.__name__, related_field, related_obj.id)
+ try:
+ setattr(related_obj, related_field, canonical)
+ related_obj.save()
+ except TypeError:
+ getattr(related_obj, related_field).add(canonical)
+ getattr(related_obj, related_field).remove(obj)
+
+
+def copy_data(canonical, obj):
+ """try to get the most data possible"""
+ for data_field in obj._meta.get_fields():
+ if not hasattr(data_field, "activitypub_field"):
+ continue
+ data_value = getattr(obj, data_field.name)
+ if not data_value:
+ continue
+ if not getattr(canonical, data_field.name):
+ print("setting data field", data_field.name, data_value)
+ setattr(canonical, data_field.name, data_value)
+ canonical.save()
+
+
+def merge_objects(canonical, obj):
+ copy_data(canonical, obj)
+ update_related(canonical, obj)
+ # remove the outdated entry
+ obj.delete()
diff --git a/bookwyrm/management/merge_command.py b/bookwyrm/management/merge_command.py
new file mode 100644
index 000000000..805dc73fa
--- /dev/null
+++ b/bookwyrm/management/merge_command.py
@@ -0,0 +1,29 @@
+from bookwyrm.management.merge import merge_objects
+from django.core.management.base import BaseCommand
+
+
+class MergeCommand(BaseCommand):
+ """base class for merge commands"""
+
+ def add_arguments(self, parser):
+ """add the arguments for this command"""
+ parser.add_argument("--canonical", type=int, required=True)
+ parser.add_argument("--other", type=int, required=True)
+
+ # pylint: disable=no-self-use,unused-argument
+ def handle(self, *args, **options):
+ """merge the two objects"""
+ model = self.MODEL
+
+ try:
+ canonical = model.objects.get(id=options["canonical"])
+ except model.DoesNotExist:
+ print("canonical book doesn’t exist!")
+ return
+ try:
+ other = model.objects.get(id=options["other"])
+ except model.DoesNotExist:
+ print("other book doesn’t exist!")
+ return
+
+ merge_objects(canonical, other)
diff --git a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py
index c06fa40a0..f25bafe15 100644
--- a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py
+++ b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py
@@ -1467,7 +1467,7 @@ class Migration(migrations.Migration):
(
"expiry",
models.DateTimeField(
- default=bookwyrm.models.site.get_passowrd_reset_expiry
+ default=bookwyrm.models.site.get_password_reset_expiry
),
),
(
diff --git a/bookwyrm/migrations/0101_auto_20210929_1847.py b/bookwyrm/migrations/0101_auto_20210929_1847.py
index 3fca28eac..967b59819 100644
--- a/bookwyrm/migrations/0101_auto_20210929_1847.py
+++ b/bookwyrm/migrations/0101_auto_20210929_1847.py
@@ -6,7 +6,7 @@ from bookwyrm.connectors.abstract_connector import infer_physical_format
def infer_format(app_registry, schema_editor):
- """set the new phsyical format field based on existing format data"""
+ """set the new physical format field based on existing format data"""
db_alias = schema_editor.connection.alias
editions = (
diff --git a/bookwyrm/migrations/0102_remove_connector_local.py b/bookwyrm/migrations/0102_remove_connector_local.py
index 857f0f589..9bfd8b1d0 100644
--- a/bookwyrm/migrations/0102_remove_connector_local.py
+++ b/bookwyrm/migrations/0102_remove_connector_local.py
@@ -5,7 +5,7 @@ from bookwyrm.settings import DOMAIN
def remove_self_connector(app_registry, schema_editor):
- """set the new phsyical format field based on existing format data"""
+ """set the new physical format field based on existing format data"""
db_alias = schema_editor.connection.alias
app_registry.get_model("bookwyrm", "Connector").objects.using(db_alias).filter(
connector_file="self_connector"
diff --git a/bookwyrm/migrations/0164_status_ready.py b/bookwyrm/migrations/0164_status_ready.py
new file mode 100644
index 000000000..fd8d49972
--- /dev/null
+++ b/bookwyrm/migrations/0164_status_ready.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-15 21:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0163_merge_0160_auto_20221101_2251_0162_importjob_task_id"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="status",
+ name="ready",
+ field=models.BooleanField(default=True),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0165_alter_inviterequest_answer.py b/bookwyrm/migrations/0165_alter_inviterequest_answer.py
new file mode 100644
index 000000000..2d2cc5e4d
--- /dev/null
+++ b/bookwyrm/migrations/0165_alter_inviterequest_answer.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-15 22:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0164_status_ready"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="inviterequest",
+ name="answer",
+ field=models.TextField(blank=True, max_length=255, null=True),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0166_sitesettings_imports_enabled.py b/bookwyrm/migrations/0166_sitesettings_imports_enabled.py
new file mode 100644
index 000000000..ccf4ef374
--- /dev/null
+++ b/bookwyrm/migrations/0166_sitesettings_imports_enabled.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-17 21:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0165_alter_inviterequest_answer"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="sitesettings",
+ name="imports_enabled",
+ field=models.BooleanField(default=True),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0167_auto_20221125_1900.py b/bookwyrm/migrations/0167_auto_20221125_1900.py
new file mode 100644
index 000000000..db258b7c5
--- /dev/null
+++ b/bookwyrm/migrations/0167_auto_20221125_1900.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-11-25 19:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0166_sitesettings_imports_enabled"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="sitesettings",
+ name="impressum",
+ field=models.TextField(default="Add a impressum here."),
+ ),
+ migrations.AddField(
+ model_name="sitesettings",
+ name="show_impressum",
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0167_sitesettings_import_size_limit.py b/bookwyrm/migrations/0167_sitesettings_import_size_limit.py
new file mode 100644
index 000000000..fdbfaf51d
--- /dev/null
+++ b/bookwyrm/migrations/0167_sitesettings_import_size_limit.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-12-05 13:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0166_sitesettings_imports_enabled"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="sitesettings",
+ name="import_size_limit",
+ field=models.IntegerField(default=0),
+ ),
+ migrations.AddField(
+ model_name="sitesettings",
+ name="import_limit_reset",
+ field=models.IntegerField(default=0),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0168_auto_20221205_1701.py b/bookwyrm/migrations/0168_auto_20221205_1701.py
new file mode 100644
index 000000000..45d6c30e7
--- /dev/null
+++ b/bookwyrm/migrations/0168_auto_20221205_1701.py
@@ -0,0 +1,28 @@
+# Generated by Django 3.2.16 on 2022-12-05 17:01
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0167_auto_20221125_1900"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="author",
+ name="aasin",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
+ ),
+ migrations.AddField(
+ model_name="book",
+ name="aasin",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0168_auto_20221205_2331.py b/bookwyrm/migrations/0168_auto_20221205_2331.py
new file mode 100644
index 000000000..901ca56f0
--- /dev/null
+++ b/bookwyrm/migrations/0168_auto_20221205_2331.py
@@ -0,0 +1,63 @@
+""" I added two new permission types and a new group to the management command that
+creates the database on install, this creates them for existing instances """
+# Generated by Django 3.2.16 on 2022-12-05 23:31
+
+from django.db import migrations
+
+
+def create_groups_and_perms(apps, schema_editor):
+ """create the new "owner" group and "system admin" permission"""
+ db_alias = schema_editor.connection.alias
+ group_model = apps.get_model("auth", "Group")
+ # Add the "owner" group, if needed
+ owner_group, group_created = group_model.objects.using(db_alias).get_or_create(
+ name="owner"
+ )
+
+ # Create perms, if needed
+ user_model = apps.get_model("bookwyrm", "User")
+ content_type_model = apps.get_model("contenttypes", "ContentType")
+ content_type = content_type_model.objects.get_for_model(user_model)
+ perms_model = apps.get_model("auth", "Permission")
+ reg_perm, perm_created = perms_model.objects.using(db_alias).get_or_create(
+ codename="manage_registration",
+ name="allow or prevent user registration",
+ content_type=content_type,
+ )
+ admin_perm, admin_perm_created = perms_model.objects.using(db_alias).get_or_create(
+ codename="system_administration",
+ name="technical controls",
+ content_type=content_type,
+ )
+
+ # Add perms to the group if anything was created
+ if group_created or perm_created or admin_perm_created:
+ perms = [
+ "edit_instance_settings",
+ "set_user_group",
+ "control_federation",
+ "create_invites",
+ "moderate_user",
+ "moderate_post",
+ "edit_book",
+ ]
+ owner_group.permissions.set(
+ perms_model.objects.using(db_alias).filter(codename__in=perms).all()
+ )
+
+ # also extend these perms to admins
+ # This is get or create so the tests don't fail -- it should already exist
+ admin_group, _ = group_model.objects.using(db_alias).get_or_create(name="admin")
+ admin_group.permissions.add(reg_perm)
+ admin_group.permissions.add(admin_perm)
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0167_auto_20221125_1900"),
+ ]
+
+ operations = [
+ migrations.RunPython(create_groups_and_perms, migrations.RunPython.noop)
+ ]
diff --git a/bookwyrm/migrations/0169_auto_20221206_0902.py b/bookwyrm/migrations/0169_auto_20221206_0902.py
new file mode 100644
index 000000000..7235490eb
--- /dev/null
+++ b/bookwyrm/migrations/0169_auto_20221206_0902.py
@@ -0,0 +1,28 @@
+# Generated by Django 3.2.16 on 2022-12-06 09:02
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0168_auto_20221205_1701"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="author",
+ name="isfdb",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
+ ),
+ migrations.AddField(
+ model_name="book",
+ name="isfdb",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902.py b/bookwyrm/migrations/0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902.py
new file mode 100644
index 000000000..3e199b014
--- /dev/null
+++ b/bookwyrm/migrations/0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902.py
@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2022-12-11 20:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0168_auto_20221205_2331"),
+ ("bookwyrm", "0169_auto_20221206_0902"),
+ ]
+
+ operations = []
diff --git a/bookwyrm/migrations/0171_alter_user_preferred_timezone.py b/bookwyrm/migrations/0171_alter_user_preferred_timezone.py
new file mode 100644
index 000000000..7dcd9546c
--- /dev/null
+++ b/bookwyrm/migrations/0171_alter_user_preferred_timezone.py
@@ -0,0 +1,631 @@
+# Generated by Django 3.2.16 on 2022-12-19 15:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="user",
+ name="preferred_timezone",
+ field=models.CharField(
+ choices=[
+ ("Africa/Abidjan", "Africa/Abidjan"),
+ ("Africa/Accra", "Africa/Accra"),
+ ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+ ("Africa/Algiers", "Africa/Algiers"),
+ ("Africa/Asmara", "Africa/Asmara"),
+ ("Africa/Asmera", "Africa/Asmera"),
+ ("Africa/Bamako", "Africa/Bamako"),
+ ("Africa/Bangui", "Africa/Bangui"),
+ ("Africa/Banjul", "Africa/Banjul"),
+ ("Africa/Bissau", "Africa/Bissau"),
+ ("Africa/Blantyre", "Africa/Blantyre"),
+ ("Africa/Brazzaville", "Africa/Brazzaville"),
+ ("Africa/Bujumbura", "Africa/Bujumbura"),
+ ("Africa/Cairo", "Africa/Cairo"),
+ ("Africa/Casablanca", "Africa/Casablanca"),
+ ("Africa/Ceuta", "Africa/Ceuta"),
+ ("Africa/Conakry", "Africa/Conakry"),
+ ("Africa/Dakar", "Africa/Dakar"),
+ ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+ ("Africa/Djibouti", "Africa/Djibouti"),
+ ("Africa/Douala", "Africa/Douala"),
+ ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+ ("Africa/Freetown", "Africa/Freetown"),
+ ("Africa/Gaborone", "Africa/Gaborone"),
+ ("Africa/Harare", "Africa/Harare"),
+ ("Africa/Johannesburg", "Africa/Johannesburg"),
+ ("Africa/Juba", "Africa/Juba"),
+ ("Africa/Kampala", "Africa/Kampala"),
+ ("Africa/Khartoum", "Africa/Khartoum"),
+ ("Africa/Kigali", "Africa/Kigali"),
+ ("Africa/Kinshasa", "Africa/Kinshasa"),
+ ("Africa/Lagos", "Africa/Lagos"),
+ ("Africa/Libreville", "Africa/Libreville"),
+ ("Africa/Lome", "Africa/Lome"),
+ ("Africa/Luanda", "Africa/Luanda"),
+ ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+ ("Africa/Lusaka", "Africa/Lusaka"),
+ ("Africa/Malabo", "Africa/Malabo"),
+ ("Africa/Maputo", "Africa/Maputo"),
+ ("Africa/Maseru", "Africa/Maseru"),
+ ("Africa/Mbabane", "Africa/Mbabane"),
+ ("Africa/Mogadishu", "Africa/Mogadishu"),
+ ("Africa/Monrovia", "Africa/Monrovia"),
+ ("Africa/Nairobi", "Africa/Nairobi"),
+ ("Africa/Ndjamena", "Africa/Ndjamena"),
+ ("Africa/Niamey", "Africa/Niamey"),
+ ("Africa/Nouakchott", "Africa/Nouakchott"),
+ ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+ ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+ ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+ ("Africa/Timbuktu", "Africa/Timbuktu"),
+ ("Africa/Tripoli", "Africa/Tripoli"),
+ ("Africa/Tunis", "Africa/Tunis"),
+ ("Africa/Windhoek", "Africa/Windhoek"),
+ ("America/Adak", "America/Adak"),
+ ("America/Anchorage", "America/Anchorage"),
+ ("America/Anguilla", "America/Anguilla"),
+ ("America/Antigua", "America/Antigua"),
+ ("America/Araguaina", "America/Araguaina"),
+ (
+ "America/Argentina/Buenos_Aires",
+ "America/Argentina/Buenos_Aires",
+ ),
+ ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+ (
+ "America/Argentina/ComodRivadavia",
+ "America/Argentina/ComodRivadavia",
+ ),
+ ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+ ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+ ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+ ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+ (
+ "America/Argentina/Rio_Gallegos",
+ "America/Argentina/Rio_Gallegos",
+ ),
+ ("America/Argentina/Salta", "America/Argentina/Salta"),
+ ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+ ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+ ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+ ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+ ("America/Aruba", "America/Aruba"),
+ ("America/Asuncion", "America/Asuncion"),
+ ("America/Atikokan", "America/Atikokan"),
+ ("America/Atka", "America/Atka"),
+ ("America/Bahia", "America/Bahia"),
+ ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+ ("America/Barbados", "America/Barbados"),
+ ("America/Belem", "America/Belem"),
+ ("America/Belize", "America/Belize"),
+ ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+ ("America/Boa_Vista", "America/Boa_Vista"),
+ ("America/Bogota", "America/Bogota"),
+ ("America/Boise", "America/Boise"),
+ ("America/Buenos_Aires", "America/Buenos_Aires"),
+ ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+ ("America/Campo_Grande", "America/Campo_Grande"),
+ ("America/Cancun", "America/Cancun"),
+ ("America/Caracas", "America/Caracas"),
+ ("America/Catamarca", "America/Catamarca"),
+ ("America/Cayenne", "America/Cayenne"),
+ ("America/Cayman", "America/Cayman"),
+ ("America/Chicago", "America/Chicago"),
+ ("America/Chihuahua", "America/Chihuahua"),
+ ("America/Ciudad_Juarez", "America/Ciudad_Juarez"),
+ ("America/Coral_Harbour", "America/Coral_Harbour"),
+ ("America/Cordoba", "America/Cordoba"),
+ ("America/Costa_Rica", "America/Costa_Rica"),
+ ("America/Creston", "America/Creston"),
+ ("America/Cuiaba", "America/Cuiaba"),
+ ("America/Curacao", "America/Curacao"),
+ ("America/Danmarkshavn", "America/Danmarkshavn"),
+ ("America/Dawson", "America/Dawson"),
+ ("America/Dawson_Creek", "America/Dawson_Creek"),
+ ("America/Denver", "America/Denver"),
+ ("America/Detroit", "America/Detroit"),
+ ("America/Dominica", "America/Dominica"),
+ ("America/Edmonton", "America/Edmonton"),
+ ("America/Eirunepe", "America/Eirunepe"),
+ ("America/El_Salvador", "America/El_Salvador"),
+ ("America/Ensenada", "America/Ensenada"),
+ ("America/Fort_Nelson", "America/Fort_Nelson"),
+ ("America/Fort_Wayne", "America/Fort_Wayne"),
+ ("America/Fortaleza", "America/Fortaleza"),
+ ("America/Glace_Bay", "America/Glace_Bay"),
+ ("America/Godthab", "America/Godthab"),
+ ("America/Goose_Bay", "America/Goose_Bay"),
+ ("America/Grand_Turk", "America/Grand_Turk"),
+ ("America/Grenada", "America/Grenada"),
+ ("America/Guadeloupe", "America/Guadeloupe"),
+ ("America/Guatemala", "America/Guatemala"),
+ ("America/Guayaquil", "America/Guayaquil"),
+ ("America/Guyana", "America/Guyana"),
+ ("America/Halifax", "America/Halifax"),
+ ("America/Havana", "America/Havana"),
+ ("America/Hermosillo", "America/Hermosillo"),
+ ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+ ("America/Indiana/Knox", "America/Indiana/Knox"),
+ ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+ ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+ ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+ ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+ ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+ ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+ ("America/Indianapolis", "America/Indianapolis"),
+ ("America/Inuvik", "America/Inuvik"),
+ ("America/Iqaluit", "America/Iqaluit"),
+ ("America/Jamaica", "America/Jamaica"),
+ ("America/Jujuy", "America/Jujuy"),
+ ("America/Juneau", "America/Juneau"),
+ ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+ ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+ ("America/Knox_IN", "America/Knox_IN"),
+ ("America/Kralendijk", "America/Kralendijk"),
+ ("America/La_Paz", "America/La_Paz"),
+ ("America/Lima", "America/Lima"),
+ ("America/Los_Angeles", "America/Los_Angeles"),
+ ("America/Louisville", "America/Louisville"),
+ ("America/Lower_Princes", "America/Lower_Princes"),
+ ("America/Maceio", "America/Maceio"),
+ ("America/Managua", "America/Managua"),
+ ("America/Manaus", "America/Manaus"),
+ ("America/Marigot", "America/Marigot"),
+ ("America/Martinique", "America/Martinique"),
+ ("America/Matamoros", "America/Matamoros"),
+ ("America/Mazatlan", "America/Mazatlan"),
+ ("America/Mendoza", "America/Mendoza"),
+ ("America/Menominee", "America/Menominee"),
+ ("America/Merida", "America/Merida"),
+ ("America/Metlakatla", "America/Metlakatla"),
+ ("America/Mexico_City", "America/Mexico_City"),
+ ("America/Miquelon", "America/Miquelon"),
+ ("America/Moncton", "America/Moncton"),
+ ("America/Monterrey", "America/Monterrey"),
+ ("America/Montevideo", "America/Montevideo"),
+ ("America/Montreal", "America/Montreal"),
+ ("America/Montserrat", "America/Montserrat"),
+ ("America/Nassau", "America/Nassau"),
+ ("America/New_York", "America/New_York"),
+ ("America/Nipigon", "America/Nipigon"),
+ ("America/Nome", "America/Nome"),
+ ("America/Noronha", "America/Noronha"),
+ ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+ ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+ (
+ "America/North_Dakota/New_Salem",
+ "America/North_Dakota/New_Salem",
+ ),
+ ("America/Nuuk", "America/Nuuk"),
+ ("America/Ojinaga", "America/Ojinaga"),
+ ("America/Panama", "America/Panama"),
+ ("America/Pangnirtung", "America/Pangnirtung"),
+ ("America/Paramaribo", "America/Paramaribo"),
+ ("America/Phoenix", "America/Phoenix"),
+ ("America/Port-au-Prince", "America/Port-au-Prince"),
+ ("America/Port_of_Spain", "America/Port_of_Spain"),
+ ("America/Porto_Acre", "America/Porto_Acre"),
+ ("America/Porto_Velho", "America/Porto_Velho"),
+ ("America/Puerto_Rico", "America/Puerto_Rico"),
+ ("America/Punta_Arenas", "America/Punta_Arenas"),
+ ("America/Rainy_River", "America/Rainy_River"),
+ ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+ ("America/Recife", "America/Recife"),
+ ("America/Regina", "America/Regina"),
+ ("America/Resolute", "America/Resolute"),
+ ("America/Rio_Branco", "America/Rio_Branco"),
+ ("America/Rosario", "America/Rosario"),
+ ("America/Santa_Isabel", "America/Santa_Isabel"),
+ ("America/Santarem", "America/Santarem"),
+ ("America/Santiago", "America/Santiago"),
+ ("America/Santo_Domingo", "America/Santo_Domingo"),
+ ("America/Sao_Paulo", "America/Sao_Paulo"),
+ ("America/Scoresbysund", "America/Scoresbysund"),
+ ("America/Shiprock", "America/Shiprock"),
+ ("America/Sitka", "America/Sitka"),
+ ("America/St_Barthelemy", "America/St_Barthelemy"),
+ ("America/St_Johns", "America/St_Johns"),
+ ("America/St_Kitts", "America/St_Kitts"),
+ ("America/St_Lucia", "America/St_Lucia"),
+ ("America/St_Thomas", "America/St_Thomas"),
+ ("America/St_Vincent", "America/St_Vincent"),
+ ("America/Swift_Current", "America/Swift_Current"),
+ ("America/Tegucigalpa", "America/Tegucigalpa"),
+ ("America/Thule", "America/Thule"),
+ ("America/Thunder_Bay", "America/Thunder_Bay"),
+ ("America/Tijuana", "America/Tijuana"),
+ ("America/Toronto", "America/Toronto"),
+ ("America/Tortola", "America/Tortola"),
+ ("America/Vancouver", "America/Vancouver"),
+ ("America/Virgin", "America/Virgin"),
+ ("America/Whitehorse", "America/Whitehorse"),
+ ("America/Winnipeg", "America/Winnipeg"),
+ ("America/Yakutat", "America/Yakutat"),
+ ("America/Yellowknife", "America/Yellowknife"),
+ ("Antarctica/Casey", "Antarctica/Casey"),
+ ("Antarctica/Davis", "Antarctica/Davis"),
+ ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+ ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+ ("Antarctica/Mawson", "Antarctica/Mawson"),
+ ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+ ("Antarctica/Palmer", "Antarctica/Palmer"),
+ ("Antarctica/Rothera", "Antarctica/Rothera"),
+ ("Antarctica/South_Pole", "Antarctica/South_Pole"),
+ ("Antarctica/Syowa", "Antarctica/Syowa"),
+ ("Antarctica/Troll", "Antarctica/Troll"),
+ ("Antarctica/Vostok", "Antarctica/Vostok"),
+ ("Arctic/Longyearbyen", "Arctic/Longyearbyen"),
+ ("Asia/Aden", "Asia/Aden"),
+ ("Asia/Almaty", "Asia/Almaty"),
+ ("Asia/Amman", "Asia/Amman"),
+ ("Asia/Anadyr", "Asia/Anadyr"),
+ ("Asia/Aqtau", "Asia/Aqtau"),
+ ("Asia/Aqtobe", "Asia/Aqtobe"),
+ ("Asia/Ashgabat", "Asia/Ashgabat"),
+ ("Asia/Ashkhabad", "Asia/Ashkhabad"),
+ ("Asia/Atyrau", "Asia/Atyrau"),
+ ("Asia/Baghdad", "Asia/Baghdad"),
+ ("Asia/Bahrain", "Asia/Bahrain"),
+ ("Asia/Baku", "Asia/Baku"),
+ ("Asia/Bangkok", "Asia/Bangkok"),
+ ("Asia/Barnaul", "Asia/Barnaul"),
+ ("Asia/Beirut", "Asia/Beirut"),
+ ("Asia/Bishkek", "Asia/Bishkek"),
+ ("Asia/Brunei", "Asia/Brunei"),
+ ("Asia/Calcutta", "Asia/Calcutta"),
+ ("Asia/Chita", "Asia/Chita"),
+ ("Asia/Choibalsan", "Asia/Choibalsan"),
+ ("Asia/Chongqing", "Asia/Chongqing"),
+ ("Asia/Chungking", "Asia/Chungking"),
+ ("Asia/Colombo", "Asia/Colombo"),
+ ("Asia/Dacca", "Asia/Dacca"),
+ ("Asia/Damascus", "Asia/Damascus"),
+ ("Asia/Dhaka", "Asia/Dhaka"),
+ ("Asia/Dili", "Asia/Dili"),
+ ("Asia/Dubai", "Asia/Dubai"),
+ ("Asia/Dushanbe", "Asia/Dushanbe"),
+ ("Asia/Famagusta", "Asia/Famagusta"),
+ ("Asia/Gaza", "Asia/Gaza"),
+ ("Asia/Harbin", "Asia/Harbin"),
+ ("Asia/Hebron", "Asia/Hebron"),
+ ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+ ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+ ("Asia/Hovd", "Asia/Hovd"),
+ ("Asia/Irkutsk", "Asia/Irkutsk"),
+ ("Asia/Istanbul", "Asia/Istanbul"),
+ ("Asia/Jakarta", "Asia/Jakarta"),
+ ("Asia/Jayapura", "Asia/Jayapura"),
+ ("Asia/Jerusalem", "Asia/Jerusalem"),
+ ("Asia/Kabul", "Asia/Kabul"),
+ ("Asia/Kamchatka", "Asia/Kamchatka"),
+ ("Asia/Karachi", "Asia/Karachi"),
+ ("Asia/Kashgar", "Asia/Kashgar"),
+ ("Asia/Kathmandu", "Asia/Kathmandu"),
+ ("Asia/Katmandu", "Asia/Katmandu"),
+ ("Asia/Khandyga", "Asia/Khandyga"),
+ ("Asia/Kolkata", "Asia/Kolkata"),
+ ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+ ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+ ("Asia/Kuching", "Asia/Kuching"),
+ ("Asia/Kuwait", "Asia/Kuwait"),
+ ("Asia/Macao", "Asia/Macao"),
+ ("Asia/Macau", "Asia/Macau"),
+ ("Asia/Magadan", "Asia/Magadan"),
+ ("Asia/Makassar", "Asia/Makassar"),
+ ("Asia/Manila", "Asia/Manila"),
+ ("Asia/Muscat", "Asia/Muscat"),
+ ("Asia/Nicosia", "Asia/Nicosia"),
+ ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+ ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+ ("Asia/Omsk", "Asia/Omsk"),
+ ("Asia/Oral", "Asia/Oral"),
+ ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+ ("Asia/Pontianak", "Asia/Pontianak"),
+ ("Asia/Pyongyang", "Asia/Pyongyang"),
+ ("Asia/Qatar", "Asia/Qatar"),
+ ("Asia/Qostanay", "Asia/Qostanay"),
+ ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+ ("Asia/Rangoon", "Asia/Rangoon"),
+ ("Asia/Riyadh", "Asia/Riyadh"),
+ ("Asia/Saigon", "Asia/Saigon"),
+ ("Asia/Sakhalin", "Asia/Sakhalin"),
+ ("Asia/Samarkand", "Asia/Samarkand"),
+ ("Asia/Seoul", "Asia/Seoul"),
+ ("Asia/Shanghai", "Asia/Shanghai"),
+ ("Asia/Singapore", "Asia/Singapore"),
+ ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+ ("Asia/Taipei", "Asia/Taipei"),
+ ("Asia/Tashkent", "Asia/Tashkent"),
+ ("Asia/Tbilisi", "Asia/Tbilisi"),
+ ("Asia/Tehran", "Asia/Tehran"),
+ ("Asia/Tel_Aviv", "Asia/Tel_Aviv"),
+ ("Asia/Thimbu", "Asia/Thimbu"),
+ ("Asia/Thimphu", "Asia/Thimphu"),
+ ("Asia/Tokyo", "Asia/Tokyo"),
+ ("Asia/Tomsk", "Asia/Tomsk"),
+ ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"),
+ ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+ ("Asia/Ulan_Bator", "Asia/Ulan_Bator"),
+ ("Asia/Urumqi", "Asia/Urumqi"),
+ ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+ ("Asia/Vientiane", "Asia/Vientiane"),
+ ("Asia/Vladivostok", "Asia/Vladivostok"),
+ ("Asia/Yakutsk", "Asia/Yakutsk"),
+ ("Asia/Yangon", "Asia/Yangon"),
+ ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+ ("Asia/Yerevan", "Asia/Yerevan"),
+ ("Atlantic/Azores", "Atlantic/Azores"),
+ ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+ ("Atlantic/Canary", "Atlantic/Canary"),
+ ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+ ("Atlantic/Faeroe", "Atlantic/Faeroe"),
+ ("Atlantic/Faroe", "Atlantic/Faroe"),
+ ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"),
+ ("Atlantic/Madeira", "Atlantic/Madeira"),
+ ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+ ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+ ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+ ("Atlantic/Stanley", "Atlantic/Stanley"),
+ ("Australia/ACT", "Australia/ACT"),
+ ("Australia/Adelaide", "Australia/Adelaide"),
+ ("Australia/Brisbane", "Australia/Brisbane"),
+ ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+ ("Australia/Canberra", "Australia/Canberra"),
+ ("Australia/Currie", "Australia/Currie"),
+ ("Australia/Darwin", "Australia/Darwin"),
+ ("Australia/Eucla", "Australia/Eucla"),
+ ("Australia/Hobart", "Australia/Hobart"),
+ ("Australia/LHI", "Australia/LHI"),
+ ("Australia/Lindeman", "Australia/Lindeman"),
+ ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+ ("Australia/Melbourne", "Australia/Melbourne"),
+ ("Australia/NSW", "Australia/NSW"),
+ ("Australia/North", "Australia/North"),
+ ("Australia/Perth", "Australia/Perth"),
+ ("Australia/Queensland", "Australia/Queensland"),
+ ("Australia/South", "Australia/South"),
+ ("Australia/Sydney", "Australia/Sydney"),
+ ("Australia/Tasmania", "Australia/Tasmania"),
+ ("Australia/Victoria", "Australia/Victoria"),
+ ("Australia/West", "Australia/West"),
+ ("Australia/Yancowinna", "Australia/Yancowinna"),
+ ("Brazil/Acre", "Brazil/Acre"),
+ ("Brazil/DeNoronha", "Brazil/DeNoronha"),
+ ("Brazil/East", "Brazil/East"),
+ ("Brazil/West", "Brazil/West"),
+ ("CET", "CET"),
+ ("CST6CDT", "CST6CDT"),
+ ("Canada/Atlantic", "Canada/Atlantic"),
+ ("Canada/Central", "Canada/Central"),
+ ("Canada/Eastern", "Canada/Eastern"),
+ ("Canada/Mountain", "Canada/Mountain"),
+ ("Canada/Newfoundland", "Canada/Newfoundland"),
+ ("Canada/Pacific", "Canada/Pacific"),
+ ("Canada/Saskatchewan", "Canada/Saskatchewan"),
+ ("Canada/Yukon", "Canada/Yukon"),
+ ("Chile/Continental", "Chile/Continental"),
+ ("Chile/EasterIsland", "Chile/EasterIsland"),
+ ("Cuba", "Cuba"),
+ ("EET", "EET"),
+ ("EST", "EST"),
+ ("EST5EDT", "EST5EDT"),
+ ("Egypt", "Egypt"),
+ ("Eire", "Eire"),
+ ("Etc/GMT", "Etc/GMT"),
+ ("Etc/GMT+0", "Etc/GMT+0"),
+ ("Etc/GMT+1", "Etc/GMT+1"),
+ ("Etc/GMT+10", "Etc/GMT+10"),
+ ("Etc/GMT+11", "Etc/GMT+11"),
+ ("Etc/GMT+12", "Etc/GMT+12"),
+ ("Etc/GMT+2", "Etc/GMT+2"),
+ ("Etc/GMT+3", "Etc/GMT+3"),
+ ("Etc/GMT+4", "Etc/GMT+4"),
+ ("Etc/GMT+5", "Etc/GMT+5"),
+ ("Etc/GMT+6", "Etc/GMT+6"),
+ ("Etc/GMT+7", "Etc/GMT+7"),
+ ("Etc/GMT+8", "Etc/GMT+8"),
+ ("Etc/GMT+9", "Etc/GMT+9"),
+ ("Etc/GMT-0", "Etc/GMT-0"),
+ ("Etc/GMT-1", "Etc/GMT-1"),
+ ("Etc/GMT-10", "Etc/GMT-10"),
+ ("Etc/GMT-11", "Etc/GMT-11"),
+ ("Etc/GMT-12", "Etc/GMT-12"),
+ ("Etc/GMT-13", "Etc/GMT-13"),
+ ("Etc/GMT-14", "Etc/GMT-14"),
+ ("Etc/GMT-2", "Etc/GMT-2"),
+ ("Etc/GMT-3", "Etc/GMT-3"),
+ ("Etc/GMT-4", "Etc/GMT-4"),
+ ("Etc/GMT-5", "Etc/GMT-5"),
+ ("Etc/GMT-6", "Etc/GMT-6"),
+ ("Etc/GMT-7", "Etc/GMT-7"),
+ ("Etc/GMT-8", "Etc/GMT-8"),
+ ("Etc/GMT-9", "Etc/GMT-9"),
+ ("Etc/GMT0", "Etc/GMT0"),
+ ("Etc/Greenwich", "Etc/Greenwich"),
+ ("Etc/UCT", "Etc/UCT"),
+ ("Etc/UTC", "Etc/UTC"),
+ ("Etc/Universal", "Etc/Universal"),
+ ("Etc/Zulu", "Etc/Zulu"),
+ ("Europe/Amsterdam", "Europe/Amsterdam"),
+ ("Europe/Andorra", "Europe/Andorra"),
+ ("Europe/Astrakhan", "Europe/Astrakhan"),
+ ("Europe/Athens", "Europe/Athens"),
+ ("Europe/Belfast", "Europe/Belfast"),
+ ("Europe/Belgrade", "Europe/Belgrade"),
+ ("Europe/Berlin", "Europe/Berlin"),
+ ("Europe/Bratislava", "Europe/Bratislava"),
+ ("Europe/Brussels", "Europe/Brussels"),
+ ("Europe/Bucharest", "Europe/Bucharest"),
+ ("Europe/Budapest", "Europe/Budapest"),
+ ("Europe/Busingen", "Europe/Busingen"),
+ ("Europe/Chisinau", "Europe/Chisinau"),
+ ("Europe/Copenhagen", "Europe/Copenhagen"),
+ ("Europe/Dublin", "Europe/Dublin"),
+ ("Europe/Gibraltar", "Europe/Gibraltar"),
+ ("Europe/Guernsey", "Europe/Guernsey"),
+ ("Europe/Helsinki", "Europe/Helsinki"),
+ ("Europe/Isle_of_Man", "Europe/Isle_of_Man"),
+ ("Europe/Istanbul", "Europe/Istanbul"),
+ ("Europe/Jersey", "Europe/Jersey"),
+ ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+ ("Europe/Kiev", "Europe/Kiev"),
+ ("Europe/Kirov", "Europe/Kirov"),
+ ("Europe/Kyiv", "Europe/Kyiv"),
+ ("Europe/Lisbon", "Europe/Lisbon"),
+ ("Europe/Ljubljana", "Europe/Ljubljana"),
+ ("Europe/London", "Europe/London"),
+ ("Europe/Luxembourg", "Europe/Luxembourg"),
+ ("Europe/Madrid", "Europe/Madrid"),
+ ("Europe/Malta", "Europe/Malta"),
+ ("Europe/Mariehamn", "Europe/Mariehamn"),
+ ("Europe/Minsk", "Europe/Minsk"),
+ ("Europe/Monaco", "Europe/Monaco"),
+ ("Europe/Moscow", "Europe/Moscow"),
+ ("Europe/Nicosia", "Europe/Nicosia"),
+ ("Europe/Oslo", "Europe/Oslo"),
+ ("Europe/Paris", "Europe/Paris"),
+ ("Europe/Podgorica", "Europe/Podgorica"),
+ ("Europe/Prague", "Europe/Prague"),
+ ("Europe/Riga", "Europe/Riga"),
+ ("Europe/Rome", "Europe/Rome"),
+ ("Europe/Samara", "Europe/Samara"),
+ ("Europe/San_Marino", "Europe/San_Marino"),
+ ("Europe/Sarajevo", "Europe/Sarajevo"),
+ ("Europe/Saratov", "Europe/Saratov"),
+ ("Europe/Simferopol", "Europe/Simferopol"),
+ ("Europe/Skopje", "Europe/Skopje"),
+ ("Europe/Sofia", "Europe/Sofia"),
+ ("Europe/Stockholm", "Europe/Stockholm"),
+ ("Europe/Tallinn", "Europe/Tallinn"),
+ ("Europe/Tirane", "Europe/Tirane"),
+ ("Europe/Tiraspol", "Europe/Tiraspol"),
+ ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+ ("Europe/Uzhgorod", "Europe/Uzhgorod"),
+ ("Europe/Vaduz", "Europe/Vaduz"),
+ ("Europe/Vatican", "Europe/Vatican"),
+ ("Europe/Vienna", "Europe/Vienna"),
+ ("Europe/Vilnius", "Europe/Vilnius"),
+ ("Europe/Volgograd", "Europe/Volgograd"),
+ ("Europe/Warsaw", "Europe/Warsaw"),
+ ("Europe/Zagreb", "Europe/Zagreb"),
+ ("Europe/Zaporozhye", "Europe/Zaporozhye"),
+ ("Europe/Zurich", "Europe/Zurich"),
+ ("GB", "GB"),
+ ("GB-Eire", "GB-Eire"),
+ ("GMT", "GMT"),
+ ("GMT+0", "GMT+0"),
+ ("GMT-0", "GMT-0"),
+ ("GMT0", "GMT0"),
+ ("Greenwich", "Greenwich"),
+ ("HST", "HST"),
+ ("Hongkong", "Hongkong"),
+ ("Iceland", "Iceland"),
+ ("Indian/Antananarivo", "Indian/Antananarivo"),
+ ("Indian/Chagos", "Indian/Chagos"),
+ ("Indian/Christmas", "Indian/Christmas"),
+ ("Indian/Cocos", "Indian/Cocos"),
+ ("Indian/Comoro", "Indian/Comoro"),
+ ("Indian/Kerguelen", "Indian/Kerguelen"),
+ ("Indian/Mahe", "Indian/Mahe"),
+ ("Indian/Maldives", "Indian/Maldives"),
+ ("Indian/Mauritius", "Indian/Mauritius"),
+ ("Indian/Mayotte", "Indian/Mayotte"),
+ ("Indian/Reunion", "Indian/Reunion"),
+ ("Iran", "Iran"),
+ ("Israel", "Israel"),
+ ("Jamaica", "Jamaica"),
+ ("Japan", "Japan"),
+ ("Kwajalein", "Kwajalein"),
+ ("Libya", "Libya"),
+ ("MET", "MET"),
+ ("MST", "MST"),
+ ("MST7MDT", "MST7MDT"),
+ ("Mexico/BajaNorte", "Mexico/BajaNorte"),
+ ("Mexico/BajaSur", "Mexico/BajaSur"),
+ ("Mexico/General", "Mexico/General"),
+ ("NZ", "NZ"),
+ ("NZ-CHAT", "NZ-CHAT"),
+ ("Navajo", "Navajo"),
+ ("PRC", "PRC"),
+ ("PST8PDT", "PST8PDT"),
+ ("Pacific/Apia", "Pacific/Apia"),
+ ("Pacific/Auckland", "Pacific/Auckland"),
+ ("Pacific/Bougainville", "Pacific/Bougainville"),
+ ("Pacific/Chatham", "Pacific/Chatham"),
+ ("Pacific/Chuuk", "Pacific/Chuuk"),
+ ("Pacific/Easter", "Pacific/Easter"),
+ ("Pacific/Efate", "Pacific/Efate"),
+ ("Pacific/Enderbury", "Pacific/Enderbury"),
+ ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+ ("Pacific/Fiji", "Pacific/Fiji"),
+ ("Pacific/Funafuti", "Pacific/Funafuti"),
+ ("Pacific/Galapagos", "Pacific/Galapagos"),
+ ("Pacific/Gambier", "Pacific/Gambier"),
+ ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+ ("Pacific/Guam", "Pacific/Guam"),
+ ("Pacific/Honolulu", "Pacific/Honolulu"),
+ ("Pacific/Johnston", "Pacific/Johnston"),
+ ("Pacific/Kanton", "Pacific/Kanton"),
+ ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+ ("Pacific/Kosrae", "Pacific/Kosrae"),
+ ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+ ("Pacific/Majuro", "Pacific/Majuro"),
+ ("Pacific/Marquesas", "Pacific/Marquesas"),
+ ("Pacific/Midway", "Pacific/Midway"),
+ ("Pacific/Nauru", "Pacific/Nauru"),
+ ("Pacific/Niue", "Pacific/Niue"),
+ ("Pacific/Norfolk", "Pacific/Norfolk"),
+ ("Pacific/Noumea", "Pacific/Noumea"),
+ ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+ ("Pacific/Palau", "Pacific/Palau"),
+ ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+ ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+ ("Pacific/Ponape", "Pacific/Ponape"),
+ ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+ ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+ ("Pacific/Saipan", "Pacific/Saipan"),
+ ("Pacific/Samoa", "Pacific/Samoa"),
+ ("Pacific/Tahiti", "Pacific/Tahiti"),
+ ("Pacific/Tarawa", "Pacific/Tarawa"),
+ ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+ ("Pacific/Truk", "Pacific/Truk"),
+ ("Pacific/Wake", "Pacific/Wake"),
+ ("Pacific/Wallis", "Pacific/Wallis"),
+ ("Pacific/Yap", "Pacific/Yap"),
+ ("Poland", "Poland"),
+ ("Portugal", "Portugal"),
+ ("ROC", "ROC"),
+ ("ROK", "ROK"),
+ ("Singapore", "Singapore"),
+ ("Turkey", "Turkey"),
+ ("UCT", "UCT"),
+ ("US/Alaska", "US/Alaska"),
+ ("US/Aleutian", "US/Aleutian"),
+ ("US/Arizona", "US/Arizona"),
+ ("US/Central", "US/Central"),
+ ("US/East-Indiana", "US/East-Indiana"),
+ ("US/Eastern", "US/Eastern"),
+ ("US/Hawaii", "US/Hawaii"),
+ ("US/Indiana-Starke", "US/Indiana-Starke"),
+ ("US/Michigan", "US/Michigan"),
+ ("US/Mountain", "US/Mountain"),
+ ("US/Pacific", "US/Pacific"),
+ ("US/Samoa", "US/Samoa"),
+ ("UTC", "UTC"),
+ ("Universal", "Universal"),
+ ("W-SU", "W-SU"),
+ ("WET", "WET"),
+ ("Zulu", "Zulu"),
+ ],
+ default="UTC",
+ max_length=255,
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0171_merge_20221219_2020.py b/bookwyrm/migrations/0171_merge_20221219_2020.py
new file mode 100644
index 000000000..53d44872f
--- /dev/null
+++ b/bookwyrm/migrations/0171_merge_20221219_2020.py
@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2022-12-19 20:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0167_sitesettings_import_size_limit"),
+ ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+ ]
+
+ operations = []
diff --git a/bookwyrm/migrations/0172_alter_user_preferred_language.py b/bookwyrm/migrations/0172_alter_user_preferred_language.py
new file mode 100644
index 000000000..2d0e033af
--- /dev/null
+++ b/bookwyrm/migrations/0172_alter_user_preferred_language.py
@@ -0,0 +1,42 @@
+# Generated by Django 3.2.16 on 2022-12-21 18:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0171_alter_user_preferred_timezone"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="user",
+ name="preferred_language",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("en-us", "English"),
+ ("ca-es", "Català (Catalan)"),
+ ("de-de", "Deutsch (German)"),
+ ("es-es", "Español (Spanish)"),
+ ("eu-es", "Euskara (Basque)"),
+ ("gl-es", "Galego (Galician)"),
+ ("it-it", "Italiano (Italian)"),
+ ("fi-fi", "Suomi (Finnish)"),
+ ("fr-fr", "Français (French)"),
+ ("lt-lt", "Lietuvių (Lithuanian)"),
+ ("no-no", "Norsk (Norwegian)"),
+ ("pl-pl", "Polski (Polish)"),
+ ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+ ("pt-pt", "Português Europeu (European Portuguese)"),
+ ("ro-ro", "Română (Romanian)"),
+ ("sv-se", "Svenska (Swedish)"),
+ ("zh-hans", "简体中文 (Simplified Chinese)"),
+ ("zh-hant", "繁體中文 (Traditional Chinese)"),
+ ],
+ max_length=255,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0173_author_website.py b/bookwyrm/migrations/0173_author_website.py
new file mode 100644
index 000000000..fda3debf1
--- /dev/null
+++ b/bookwyrm/migrations/0173_author_website.py
@@ -0,0 +1,21 @@
+# Generated by Django 3.2.16 on 2023-01-15 08:38
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0172_alter_user_preferred_language"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="author",
+ name="website",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0173_default_user_auth_group_setting.py b/bookwyrm/migrations/0173_default_user_auth_group_setting.py
new file mode 100644
index 000000000..1f7e26612
--- /dev/null
+++ b/bookwyrm/migrations/0173_default_user_auth_group_setting.py
@@ -0,0 +1,34 @@
+# Generated by Django 3.2.16 on 2022-12-27 21:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def backfill_sitesettings(apps, schema_editor):
+ db_alias = schema_editor.connection.alias
+ group_model = apps.get_model("auth", "Group")
+ editor_group = group_model.objects.using(db_alias).filter(name="editor").first()
+
+ sitesettings_model = apps.get_model("bookwyrm", "SiteSettings")
+ sitesettings_model.objects.update(default_user_auth_group=editor_group)
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0175_merge_0173_author_website_0174_merge_20230111_1523"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="sitesettings",
+ name="default_user_auth_group",
+ field=models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.RESTRICT,
+ to="auth.group",
+ ),
+ ),
+ migrations.RunPython(backfill_sitesettings, migrations.RunPython.noop),
+ ]
diff --git a/bookwyrm/migrations/0173_merge_20230102_1444.py b/bookwyrm/migrations/0173_merge_20230102_1444.py
new file mode 100644
index 000000000..c3e37a76f
--- /dev/null
+++ b/bookwyrm/migrations/0173_merge_20230102_1444.py
@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2023-01-02 14:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0171_merge_20221219_2020"),
+ ("bookwyrm", "0172_alter_user_preferred_language"),
+ ]
+
+ operations = []
diff --git a/bookwyrm/migrations/0174_auto_20230130_1240.py b/bookwyrm/migrations/0174_auto_20230130_1240.py
new file mode 100644
index 000000000..7337b6b46
--- /dev/null
+++ b/bookwyrm/migrations/0174_auto_20230130_1240.py
@@ -0,0 +1,35 @@
+# Generated by Django 3.2.16 on 2023-01-30 12:40
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("auth", "0012_alter_user_first_name_max_length"),
+ ("bookwyrm", "0173_default_user_auth_group_setting"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="quotation",
+ name="endposition",
+ field=models.IntegerField(
+ blank=True,
+ null=True,
+ validators=[django.core.validators.MinValueValidator(0)],
+ ),
+ ),
+ migrations.AlterField(
+ model_name="sitesettings",
+ name="default_user_auth_group",
+ field=models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.PROTECT,
+ to="auth.group",
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0174_auto_20230222_1742.py b/bookwyrm/migrations/0174_auto_20230222_1742.py
new file mode 100644
index 000000000..f30d61a46
--- /dev/null
+++ b/bookwyrm/migrations/0174_auto_20230222_1742.py
@@ -0,0 +1,46 @@
+# Generated by Django 3.2.18 on 2023-02-22 17:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0174_auto_20230130_1240"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="notification",
+ name="related_link_domains",
+ field=models.ManyToManyField(to="bookwyrm.LinkDomain"),
+ ),
+ migrations.AlterField(
+ model_name="notification",
+ name="notification_type",
+ field=models.CharField(
+ choices=[
+ ("FAVORITE", "Favorite"),
+ ("REPLY", "Reply"),
+ ("MENTION", "Mention"),
+ ("TAG", "Tag"),
+ ("FOLLOW", "Follow"),
+ ("FOLLOW_REQUEST", "Follow Request"),
+ ("BOOST", "Boost"),
+ ("IMPORT", "Import"),
+ ("ADD", "Add"),
+ ("REPORT", "Report"),
+ ("LINK_DOMAIN", "Link Domain"),
+ ("INVITE", "Invite"),
+ ("ACCEPT", "Accept"),
+ ("JOIN", "Join"),
+ ("LEAVE", "Leave"),
+ ("REMOVE", "Remove"),
+ ("GROUP_PRIVACY", "Group Privacy"),
+ ("GROUP_NAME", "Group Name"),
+ ("GROUP_DESCRIPTION", "Group Description"),
+ ],
+ max_length=255,
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0174_merge_20230111_1523.py b/bookwyrm/migrations/0174_merge_20230111_1523.py
new file mode 100644
index 000000000..fd57083f6
--- /dev/null
+++ b/bookwyrm/migrations/0174_merge_20230111_1523.py
@@ -0,0 +1,12 @@
+# Generated by Django 3.2.16 on 2023-01-11 15:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0173_merge_20230102_1444"),
+ ]
+
+ operations = []
diff --git a/bookwyrm/migrations/0175_merge_0173_author_website_0174_merge_20230111_1523.py b/bookwyrm/migrations/0175_merge_0173_author_website_0174_merge_20230111_1523.py
new file mode 100644
index 000000000..a215076b4
--- /dev/null
+++ b/bookwyrm/migrations/0175_merge_0173_author_website_0174_merge_20230111_1523.py
@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2023-01-19 20:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0173_author_website"),
+ ("bookwyrm", "0174_merge_20230111_1523"),
+ ]
+
+ operations = []
diff --git a/bookwyrm/migrations/0176_hashtag_support.py b/bookwyrm/migrations/0176_hashtag_support.py
new file mode 100644
index 000000000..96e79ff36
--- /dev/null
+++ b/bookwyrm/migrations/0176_hashtag_support.py
@@ -0,0 +1,53 @@
+# Generated by Django 3.2.16 on 2022-12-17 19:28
+
+import bookwyrm.models.fields
+import django.contrib.postgres.fields.citext
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0174_auto_20230130_1240"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="Hashtag",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("created_date", models.DateTimeField(auto_now_add=True)),
+ ("updated_date", models.DateTimeField(auto_now=True)),
+ (
+ "remote_id",
+ bookwyrm.models.fields.RemoteIdField(
+ max_length=255,
+ null=True,
+ validators=[bookwyrm.models.fields.validate_remote_id],
+ ),
+ ),
+ (
+ "name",
+ django.contrib.postgres.fields.citext.CICharField(max_length=256),
+ ),
+ ],
+ options={
+ "abstract": False,
+ },
+ ),
+ migrations.AddField(
+ model_name="status",
+ name="mention_hashtags",
+ field=bookwyrm.models.fields.TagField(
+ related_name="mention_hashtag", to="bookwyrm.Hashtag"
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0177_merge_0174_auto_20230222_1742_0176_hashtag_support.py b/bookwyrm/migrations/0177_merge_0174_auto_20230222_1742_0176_hashtag_support.py
new file mode 100644
index 000000000..65ace3059
--- /dev/null
+++ b/bookwyrm/migrations/0177_merge_0174_auto_20230222_1742_0176_hashtag_support.py
@@ -0,0 +1,13 @@
+# Generated by Django 3.2.18 on 2023-03-12 23:41
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0174_auto_20230222_1742"),
+ ("bookwyrm", "0176_hashtag_support"),
+ ]
+
+ operations = []
diff --git a/bookwyrm/migrations/0178_auto_20230328_2132.py b/bookwyrm/migrations/0178_auto_20230328_2132.py
new file mode 100644
index 000000000..9decc001f
--- /dev/null
+++ b/bookwyrm/migrations/0178_auto_20230328_2132.py
@@ -0,0 +1,61 @@
+# Generated by Django 3.2.18 on 2023-03-28 21:32
+
+import bookwyrm.models.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("auth", "0012_alter_user_first_name_max_length"),
+ ("bookwyrm", "0177_merge_0174_auto_20230222_1742_0176_hashtag_support"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="hashtag",
+ name="name",
+ field=bookwyrm.models.fields.CICharField(max_length=256),
+ ),
+ migrations.AlterField(
+ model_name="sitesettings",
+ name="default_user_auth_group",
+ field=models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.RESTRICT,
+ to="auth.group",
+ ),
+ ),
+ migrations.AlterField(
+ model_name="user",
+ name="preferred_language",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("en-us", "English"),
+ ("ca-es", "Català (Catalan)"),
+ ("de-de", "Deutsch (German)"),
+ ("eo-uy", "Esperanto (Esperanto)"),
+ ("es-es", "Español (Spanish)"),
+ ("eu-es", "Euskara (Basque)"),
+ ("gl-es", "Galego (Galician)"),
+ ("it-it", "Italiano (Italian)"),
+ ("fi-fi", "Suomi (Finnish)"),
+ ("fr-fr", "Français (French)"),
+ ("lt-lt", "Lietuvių (Lithuanian)"),
+ ("no-no", "Norsk (Norwegian)"),
+ ("pl-pl", "Polski (Polish)"),
+ ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+ ("pt-pt", "Português Europeu (European Portuguese)"),
+ ("ro-ro", "Română (Romanian)"),
+ ("sv-se", "Svenska (Swedish)"),
+ ("zh-hans", "简体中文 (Simplified Chinese)"),
+ ("zh-hant", "繁體中文 (Traditional Chinese)"),
+ ],
+ max_length=255,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0179_populate_sort_title.py b/bookwyrm/migrations/0179_populate_sort_title.py
new file mode 100644
index 000000000..e238bca1d
--- /dev/null
+++ b/bookwyrm/migrations/0179_populate_sort_title.py
@@ -0,0 +1,49 @@
+import re
+from itertools import chain
+
+from django.db import migrations, transaction
+from django.db.models import Q
+
+from bookwyrm.settings import LANGUAGE_ARTICLES
+
+
+def set_sort_title(edition):
+ articles = chain(
+ *(LANGUAGE_ARTICLES.get(language, ()) for language in tuple(edition.languages))
+ )
+ edition.sort_title = re.sub(
+ f'^{" |^".join(articles)} ', "", str(edition.title).lower()
+ )
+ return edition
+
+
+@transaction.atomic
+def populate_sort_title(apps, schema_editor):
+ Edition = apps.get_model("bookwyrm", "Edition")
+ db_alias = schema_editor.connection.alias
+ editions_wo_sort_title = Edition.objects.using(db_alias).filter(
+ Q(sort_title__isnull=True) | Q(sort_title__exact="")
+ )
+ batch_size = 1000
+ start = 0
+ end = batch_size
+ while True:
+ batch = editions_wo_sort_title[start:end]
+ if not batch.exists():
+ break
+ Edition.objects.bulk_update(
+ (set_sort_title(edition) for edition in batch), ["sort_title"]
+ )
+ start = end
+ end += batch_size
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0178_auto_20230328_2132"),
+ ]
+
+ operations = [
+ migrations.RunPython(populate_sort_title),
+ ]
diff --git a/bookwyrm/migrations/0179_reportcomment_comment_type.py b/bookwyrm/migrations/0179_reportcomment_comment_type.py
new file mode 100644
index 000000000..a8a446096
--- /dev/null
+++ b/bookwyrm/migrations/0179_reportcomment_comment_type.py
@@ -0,0 +1,36 @@
+# Generated by Django 3.2.18 on 2023-05-16 16:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0178_auto_20230328_2132"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="reportcomment",
+ name="action_type",
+ field=models.CharField(
+ choices=[
+ ("comment", "Comment"),
+ ("resolve", "Resolved report"),
+ ("reopen", "Re-opened report"),
+ ("message_reporter", "Messaged reporter"),
+ ("message_offender", "Messaged reported user"),
+ ("user_suspension", "Suspended user"),
+ ("user_unsuspension", "Un-suspended user"),
+ ("user_perms", "Changed user permission level"),
+ ("user_deletion", "Deleted user account"),
+ ("block_domain", "Blocked domain"),
+ ("approve_domain", "Approved domain"),
+ ("delete_item", "Deleted item"),
+ ],
+ default="comment",
+ max_length=20,
+ ),
+ ),
+ migrations.RenameModel("ReportComment", "ReportAction"),
+ ]
diff --git a/bookwyrm/migrations/0180_alter_reportaction_options.py b/bookwyrm/migrations/0180_alter_reportaction_options.py
new file mode 100644
index 000000000..2979d266e
--- /dev/null
+++ b/bookwyrm/migrations/0180_alter_reportaction_options.py
@@ -0,0 +1,17 @@
+# Generated by Django 3.2.18 on 2023-06-21 22:01
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0179_reportcomment_comment_type"),
+ ]
+
+ operations = [
+ migrations.AlterModelOptions(
+ name="reportaction",
+ options={"ordering": ("created_date",)},
+ ),
+ ]
diff --git a/bookwyrm/migrations/0180_alter_user_preferred_language.py b/bookwyrm/migrations/0180_alter_user_preferred_language.py
new file mode 100644
index 000000000..b4ab996ec
--- /dev/null
+++ b/bookwyrm/migrations/0180_alter_user_preferred_language.py
@@ -0,0 +1,44 @@
+# Generated by Django 3.2.19 on 2023-07-23 19:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0179_populate_sort_title"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="user",
+ name="preferred_language",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("en-us", "English"),
+ ("ca-es", "Català (Catalan)"),
+ ("de-de", "Deutsch (German)"),
+ ("eo-uy", "Esperanto (Esperanto)"),
+ ("es-es", "Español (Spanish)"),
+ ("eu-es", "Euskara (Basque)"),
+ ("gl-es", "Galego (Galician)"),
+ ("it-it", "Italiano (Italian)"),
+ ("fi-fi", "Suomi (Finnish)"),
+ ("fr-fr", "Français (French)"),
+ ("lt-lt", "Lietuvių (Lithuanian)"),
+ ("nl-nl", "Nederlands (Dutch)"),
+ ("no-no", "Norsk (Norwegian)"),
+ ("pl-pl", "Polski (Polish)"),
+ ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+ ("pt-pt", "Português Europeu (European Portuguese)"),
+ ("ro-ro", "Română (Romanian)"),
+ ("sv-se", "Svenska (Swedish)"),
+ ("zh-hans", "简体中文 (Simplified Chinese)"),
+ ("zh-hant", "繁體中文 (Traditional Chinese)"),
+ ],
+ max_length=255,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/bookwyrm/migrations/0181_merge_20230806_2302.py b/bookwyrm/migrations/0181_merge_20230806_2302.py
new file mode 100644
index 000000000..f4f05b886
--- /dev/null
+++ b/bookwyrm/migrations/0181_merge_20230806_2302.py
@@ -0,0 +1,13 @@
+# Generated by Django 3.2.20 on 2023-08-06 23:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0180_alter_reportaction_options"),
+ ("bookwyrm", "0180_alter_user_preferred_language"),
+ ]
+
+ operations = []
diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py
index ae7000162..7b779190b 100644
--- a/bookwyrm/models/__init__.py
+++ b/bookwyrm/models/__init__.py
@@ -20,7 +20,7 @@ from .readthrough import ReadThrough, ProgressUpdate, ProgressMode
from .user import User, KeyPair
from .annual_goal import AnnualGoal
from .relationship import UserFollows, UserFollowRequest, UserBlocks
-from .report import Report, ReportComment
+from .report import Report, ReportAction
from .federated_server import FederatedServer
from .group import Group, GroupMember, GroupMemberInvitation
@@ -34,6 +34,8 @@ from .antispam import EmailBlocklist, IPBlocklist, AutoMod, automod_task
from .notification import Notification
+from .hashtag import Hashtag
+
cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
activity_models = {
c[1].activity_serializer.__name__: c[1]
diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py
index a9c6328fb..36317ad4e 100644
--- a/bookwyrm/models/activitypub_mixin.py
+++ b/bookwyrm/models/activitypub_mixin.py
@@ -6,8 +6,9 @@ from functools import reduce
import json
import operator
import logging
-from typing import List
+from typing import Any, Optional
from uuid import uuid4
+from typing_extensions import Self
import aiohttp
from Crypto.PublicKey import RSA
@@ -21,11 +22,11 @@ from django.utils.http import http_date
from bookwyrm import activitypub
from bookwyrm.settings import USER_AGENT, PAGE_LENGTH
from bookwyrm.signatures import make_signature, make_digest
-from bookwyrm.tasks import app, MEDIUM
+from bookwyrm.tasks import app, BROADCAST
from bookwyrm.models.fields import ImageField, ManyToManyField
logger = logging.getLogger(__name__)
-# I tried to separate these classes into mutliple files but I kept getting
+# I tried to separate these classes into multiple files but I kept getting
# circular import errors so I gave up. I'm sure it could be done though!
PropertyField = namedtuple("PropertyField", ("set_activity_from_field"))
@@ -85,13 +86,13 @@ class ActivitypubMixin:
super().__init__(*args, **kwargs)
@classmethod
- def find_existing_by_remote_id(cls, remote_id):
+ def find_existing_by_remote_id(cls, remote_id: str) -> Self:
"""look up a remote id in the db"""
return cls.find_existing({"id": remote_id})
@classmethod
def find_existing(cls, data):
- """compare data to fields that can be used for deduplation.
+ """compare data to fields that can be used for deduplication.
This always includes remote_id, but can also be unique identifiers
like an isbn for an edition"""
filters = []
@@ -126,7 +127,7 @@ class ActivitypubMixin:
# there OUGHT to be only one match
return match.first()
- def broadcast(self, activity, sender, software=None, queue=MEDIUM):
+ def broadcast(self, activity, sender, software=None, queue=BROADCAST):
"""send out an activity"""
broadcast_task.apply_async(
args=(
@@ -137,7 +138,7 @@ class ActivitypubMixin:
queue=queue,
)
- def get_recipients(self, software=None) -> List[str]:
+ def get_recipients(self, software=None) -> list[str]:
"""figure out which inbox urls to post to"""
# first we have to figure out who should receive this activity
privacy = self.privacy if hasattr(self, "privacy") else "public"
@@ -198,7 +199,14 @@ class ActivitypubMixin:
class ObjectMixin(ActivitypubMixin):
"""add this mixin for object models that are AP serializable"""
- def save(self, *args, created=None, software=None, priority=MEDIUM, **kwargs):
+ def save(
+ self,
+ *args: Any,
+ created: Optional[bool] = None,
+ software: Any = None,
+ priority: str = BROADCAST,
+ **kwargs: Any,
+ ) -> None:
"""broadcast created/updated/deleted objects as appropriate"""
broadcast = kwargs.get("broadcast", True)
# this bonus kwarg would cause an error in the base save method
@@ -234,8 +242,8 @@ class ObjectMixin(ActivitypubMixin):
activity = self.to_create_activity(user)
self.broadcast(activity, user, software=software, queue=priority)
except AttributeError:
- # janky as heck, this catches the mutliple inheritence chain
- # for boosts and ignores this auxilliary broadcast
+ # janky as heck, this catches the multiple inheritance chain
+ # for boosts and ignores this auxiliary broadcast
return
return
@@ -311,7 +319,7 @@ class OrderedCollectionPageMixin(ObjectMixin):
@property
def collection_remote_id(self):
- """this can be overriden if there's a special remote id, ie outbox"""
+ """this can be overridden if there's a special remote id, ie outbox"""
return self.remote_id
def to_ordered_collection(
@@ -339,7 +347,7 @@ class OrderedCollectionPageMixin(ObjectMixin):
activity["id"] = remote_id
paginated = Paginator(queryset, PAGE_LENGTH)
- # add computed fields specific to orderd collections
+ # add computed fields specific to ordered collections
activity["totalItems"] = paginated.count
activity["first"] = f"{remote_id}?page=1"
activity["last"] = f"{remote_id}?page={paginated.num_pages}"
@@ -379,7 +387,7 @@ class CollectionItemMixin(ActivitypubMixin):
activity_serializer = activitypub.CollectionItem
- def broadcast(self, activity, sender, software="bookwyrm", queue=MEDIUM):
+ def broadcast(self, activity, sender, software="bookwyrm", queue=BROADCAST):
"""only send book collection updates to other bookwyrm instances"""
super().broadcast(activity, sender, software=software, queue=queue)
@@ -400,12 +408,12 @@ class CollectionItemMixin(ActivitypubMixin):
return []
return [collection_field.user]
- def save(self, *args, broadcast=True, priority=MEDIUM, **kwargs):
+ def save(self, *args, broadcast=True, priority=BROADCAST, **kwargs):
"""broadcast updated"""
# first off, we want to save normally no matter what
super().save(*args, **kwargs)
- # list items can be updateda, normally you would only broadcast on created
+ # list items can be updated, normally you would only broadcast on created
if not broadcast or not self.user.local:
return
@@ -444,7 +452,7 @@ class CollectionItemMixin(ActivitypubMixin):
class ActivityMixin(ActivitypubMixin):
"""add this mixin for models that are AP serializable"""
- def save(self, *args, broadcast=True, priority=MEDIUM, **kwargs):
+ def save(self, *args, broadcast=True, priority=BROADCAST, **kwargs):
"""broadcast activity"""
super().save(*args, **kwargs)
user = self.user if hasattr(self, "user") else self.user_subject
@@ -506,15 +514,15 @@ def unfurl_related_field(related_field, sort_field=None):
return related_field.remote_id
-@app.task(queue=MEDIUM)
-def broadcast_task(sender_id: int, activity: str, recipients: List[str]):
+@app.task(queue=BROADCAST)
+def broadcast_task(sender_id: int, activity: str, recipients: list[str]):
"""the celery task for broadcast"""
user_model = apps.get_model("bookwyrm.User", require_ready=True)
sender = user_model.objects.select_related("key_pair").get(id=sender_id)
asyncio.run(async_broadcast(recipients, sender, activity))
-async def async_broadcast(recipients: List[str], sender, data: str):
+async def async_broadcast(recipients: list[str], sender, data: str):
"""Send all the broadcasts simultaneously"""
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession(timeout=timeout) as session:
@@ -529,7 +537,7 @@ async def async_broadcast(recipients: List[str], sender, data: str):
async def sign_and_send(
- session: aiohttp.ClientSession, sender, data: str, destination: str
+ session: aiohttp.ClientSession, sender, data: str, destination: str, **kwargs
):
"""Sign the messages and send them in an asynchronous bundle"""
now = http_date()
@@ -539,11 +547,19 @@ async def sign_and_send(
raise ValueError("No private key found for sender")
digest = make_digest(data)
+ signature = make_signature(
+ "post",
+ sender,
+ destination,
+ now,
+ digest=digest,
+ use_legacy_key=kwargs.get("use_legacy_key"),
+ )
headers = {
"Date": now,
"Digest": digest,
- "Signature": make_signature(sender, destination, now, digest),
+ "Signature": signature,
"Content-Type": "application/activity+json; charset=utf-8",
"User-Agent": USER_AGENT,
}
@@ -554,6 +570,14 @@ async def sign_and_send(
logger.exception(
"Failed to send broadcast to %s: %s", destination, response.reason
)
+ if kwargs.get("use_legacy_key") is not True:
+ logger.info("Trying again with legacy keyId header value")
+ asyncio.ensure_future(
+ sign_and_send(
+ session, sender, data, destination, use_legacy_key=True
+ )
+ )
+
return response
except asyncio.TimeoutError:
logger.info("Connection timed out for url: %s", destination)
@@ -565,7 +589,7 @@ async def sign_and_send(
def to_ordered_collection_page(
queryset, remote_id, id_only=False, page=1, pure=False, **kwargs
):
- """serialize and pagiante a queryset"""
+ """serialize and paginate a queryset"""
paginated = Paginator(queryset, PAGE_LENGTH)
activity_page = paginated.get_page(page)
diff --git a/bookwyrm/models/annual_goal.py b/bookwyrm/models/annual_goal.py
index 53c041141..d36b822df 100644
--- a/bookwyrm/models/annual_goal.py
+++ b/bookwyrm/models/annual_goal.py
@@ -24,7 +24,7 @@ class AnnualGoal(BookWyrmModel):
)
class Meta:
- """unqiueness constraint"""
+ """uniqueness constraint"""
unique_together = ("user", "year")
@@ -52,7 +52,7 @@ class AnnualGoal(BookWyrmModel):
user=self.user,
book__in=book_ids,
)
- return {r.book.id: r.rating for r in reviews}
+ return {r.book_id: r.rating for r in reviews}
@property
def progress(self):
diff --git a/bookwyrm/models/antispam.py b/bookwyrm/models/antispam.py
index 1e20df340..94d978ec4 100644
--- a/bookwyrm/models/antispam.py
+++ b/bookwyrm/models/antispam.py
@@ -8,7 +8,7 @@ from django.db import models, transaction
from django.db.models import Q
from django.utils.translation import gettext_lazy as _
-from bookwyrm.tasks import app, LOW
+from bookwyrm.tasks import app, MISC
from .base_model import BookWyrmModel
from .user import User
@@ -65,7 +65,7 @@ class AutoMod(AdminModel):
created_by = models.ForeignKey("User", on_delete=models.PROTECT)
-@app.task(queue=LOW)
+@app.task(queue=MISC)
def automod_task():
"""Create reports"""
if not AutoMod.objects.exists():
diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index 7d2a0e62b..981e3c0cc 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -1,8 +1,8 @@
""" database schema for info about authors """
import re
+from typing import Tuple, Any
+
from django.contrib.postgres.indexes import GinIndex
-from django.core.cache import cache
-from django.core.cache.utils import make_template_fragment_key
from django.db import models
from bookwyrm import activitypub
@@ -24,6 +24,13 @@ class Author(BookDataModel):
gutenberg_id = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
+ isfdb = fields.CharField(
+ max_length=255, blank=True, null=True, deduplication_field=True
+ )
+
+ website = fields.CharField(
+ max_length=255, blank=True, null=True, deduplication_field=True
+ )
# idk probably other keys would be useful here?
born = fields.DateTimeField(blank=True, null=True)
died = fields.DateTimeField(blank=True, null=True)
@@ -33,17 +40,8 @@ class Author(BookDataModel):
)
bio = fields.HtmlField(null=True, blank=True)
- def save(self, *args, **kwargs):
- """clear related template caches"""
- # clear template caches
- if self.id:
- cache_keys = [
- make_template_fragment_key("titleby", [book])
- for book in self.book_set.values_list("id", flat=True)
- ]
- cache.delete_many(cache_keys)
-
- # normalize isni format
+ def save(self, *args: Tuple[Any, ...], **kwargs: dict[str, Any]) -> None:
+ """normalize isni format"""
if self.isni:
self.isni = re.sub(r"\s", "", self.isni)
@@ -60,6 +58,11 @@ class Author(BookDataModel):
"""generate the url from the openlibrary id"""
return f"https://openlibrary.org/authors/{self.openlibrary_key}"
+ @property
+ def isfdb_link(self):
+ """generate the url from the isni id"""
+ return f"https://www.isfdb.org/cgi-bin/ea.cgi?{self.isfdb}"
+
def get_remote_id(self):
"""editions and works both use "book" instead of model_name"""
return f"https://{DOMAIN}/author/{self.id}"
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index 5bef5c1ee..d0c3c7fd3 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -1,10 +1,11 @@
""" database schema for books and shelves """
+from itertools import chain
import re
+from typing import Any
from django.contrib.postgres.search import SearchVectorField
from django.contrib.postgres.indexes import GinIndex
from django.core.cache import cache
-from django.core.cache.utils import make_template_fragment_key
from django.db import models, transaction
from django.db.models import Prefetch
from django.dispatch import receiver
@@ -14,10 +15,12 @@ from model_utils.managers import InheritanceManager
from imagekit.models import ImageSpecField
from bookwyrm import activitypub
+from bookwyrm.isbn.isbn import hyphenator_singleton as hyphenator
from bookwyrm.preview_images import generate_edition_preview_image_task
from bookwyrm.settings import (
DOMAIN,
DEFAULT_LANGUAGE,
+ LANGUAGE_ARTICLES,
ENABLE_PREVIEW_IMAGES,
ENABLE_THUMBNAIL_GENERATION,
)
@@ -55,6 +58,12 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
asin = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
+ aasin = fields.CharField(
+ max_length=255, blank=True, null=True, deduplication_field=True
+ )
+ isfdb = fields.CharField(
+ max_length=255, blank=True, null=True, deduplication_field=True
+ )
search_vector = SearchVectorField(null=True)
last_edited_by = fields.ForeignKey(
@@ -73,12 +82,17 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
"""generate the url from the inventaire id"""
return f"https://inventaire.io/entity/{self.inventaire_id}"
+ @property
+ def isfdb_link(self):
+ """generate the url from the isfdb id"""
+ return f"https://www.isfdb.org/cgi-bin/title.cgi?{self.isfdb}"
+
class Meta:
"""can't initialize this model, that wouldn't make sense"""
abstract = True
- def save(self, *args, **kwargs):
+ def save(self, *args: Any, **kwargs: Any) -> None:
"""ensure that the remote_id is within this instance"""
if self.id:
self.remote_id = self.get_remote_id()
@@ -192,21 +206,24 @@ class Book(BookDataModel):
text += f" ({self.edition_info})"
return text
- def save(self, *args, **kwargs):
+ def save(self, *args: Any, **kwargs: Any) -> None:
"""can't be abstract for query reasons, but you shouldn't USE it"""
if not isinstance(self, Edition) and not isinstance(self, Work):
raise ValueError("Books should be added as Editions or Works")
- # clear template caches
- cache_key = make_template_fragment_key("titleby", [self.id])
- cache.delete(cache_key)
-
return super().save(*args, **kwargs)
def get_remote_id(self):
"""editions and works both use "book" instead of model_name"""
return f"https://{DOMAIN}/book/{self.id}"
+ def guess_sort_title(self):
+ """Get a best-guess sort title for the current book"""
+ articles = chain(
+ *(LANGUAGE_ARTICLES.get(language, ()) for language in tuple(self.languages))
+ )
+ return re.sub(f'^{" |^".join(articles)} ', "", str(self.title).lower())
+
def __repr__(self):
# pylint: disable=consider-using-f-string
return "<{} key={!r} title={!r}>".format(
@@ -312,10 +329,15 @@ class Edition(Book):
serialize_reverse_fields = [("file_links", "fileLinks", "-created_date")]
deserialize_reverse_fields = [("file_links", "fileLinks")]
+ @property
+ def hyphenated_isbn13(self):
+ """generate the hyphenated version of the ISBN-13"""
+ return hyphenator.hyphenate(self.isbn_13)
+
def get_rank(self):
"""calculate how complete the data is on this edition"""
rank = 0
- # big ups for havinga cover
+ # big ups for having a cover
rank += int(bool(self.cover)) * 3
# is it in the instance's preferred language?
rank += int(bool(DEFAULT_LANGUAGE in self.languages))
@@ -335,7 +357,7 @@ class Edition(Book):
# max rank is 9
return rank
- def save(self, *args, **kwargs):
+ def save(self, *args: Any, **kwargs: Any) -> None:
"""set some fields on the edition object"""
# calculate isbn 10/13
if self.isbn_13 and self.isbn_13[:3] == "978" and not self.isbn_10:
@@ -357,8 +379,25 @@ class Edition(Book):
for author_id in self.authors.values_list("id", flat=True):
cache.delete(f"author-books-{author_id}")
+ # Create sort title by removing articles from title
+ if self.sort_title in [None, ""]:
+ self.sort_title = self.guess_sort_title()
+
return super().save(*args, **kwargs)
+ @transaction.atomic
+ def repair(self):
+ """If an edition is in a bad state (missing a work), let's fix that"""
+ # made sure it actually NEEDS reapir
+ if self.parent_work:
+ return
+
+ new_work = Work.objects.create(title=self.title)
+ new_work.authors.set(self.authors.all())
+
+ self.parent_work = new_work
+ self.save(update_fields=["parent_work"], broadcast=False)
+
@classmethod
def viewer_aware_objects(cls, viewer):
"""annotate a book query with metadata related to the user"""
diff --git a/bookwyrm/models/favorite.py b/bookwyrm/models/favorite.py
index 4c3675219..98fbce550 100644
--- a/bookwyrm/models/favorite.py
+++ b/bookwyrm/models/favorite.py
@@ -20,8 +20,9 @@ class Favorite(ActivityMixin, BookWyrmModel):
activity_serializer = activitypub.Like
+ # pylint: disable=unused-argument
@classmethod
- def ignore_activity(cls, activity):
+ def ignore_activity(cls, activity, allow_external_connections=True):
"""don't bother with incoming favs of unknown statuses"""
return not Status.objects.filter(remote_id=activity.object).exists()
diff --git a/bookwyrm/models/federated_server.py b/bookwyrm/models/federated_server.py
index eb03d457e..e1081ed45 100644
--- a/bookwyrm/models/federated_server.py
+++ b/bookwyrm/models/federated_server.py
@@ -61,7 +61,7 @@ class FederatedServer(BookWyrmModel):
).update(active=True, deactivation_reason=None)
@classmethod
- def is_blocked(cls, url):
+ def is_blocked(cls, url: str) -> bool:
"""look up if a domain is blocked"""
url = urlparse(url)
domain = url.netloc
diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py
index 785f3397c..28effaf9b 100644
--- a/bookwyrm/models/fields.py
+++ b/bookwyrm/models/fields.py
@@ -1,5 +1,6 @@
""" activitypub-aware django model fields """
from dataclasses import MISSING
+from datetime import datetime
import re
from uuid import uuid4
from urllib.parse import urljoin
@@ -7,12 +8,14 @@ from urllib.parse import urljoin
import dateutil.parser
from dateutil.parser import ParserError
from django.contrib.postgres.fields import ArrayField as DjangoArrayField
+from django.contrib.postgres.fields import CICharField as DjangoCICharField
from django.core.exceptions import ValidationError
from django.db import models
from django.forms import ClearableFileInput, ImageField as DjangoImageField
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from django.utils.encoding import filepath_to_uri
+from markdown import markdown
from bookwyrm import activitypub
from bookwyrm.connectors import get_image
@@ -66,16 +69,20 @@ class ActivitypubFieldMixin:
self.activitypub_field = activitypub_field
super().__init__(*args, **kwargs)
- def set_field_from_activity(self, instance, data, overwrite=True):
- """helper function for assinging a value to the field. Returns if changed"""
+ def set_field_from_activity(
+ self, instance, data, overwrite=True, allow_external_connections=True
+ ):
+ """helper function for assigning a value to the field. Returns if changed"""
try:
value = getattr(data, self.get_activitypub_field())
except AttributeError:
- # masssively hack-y workaround for boosts
+ # massively hack-y workaround for boosts
if self.get_activitypub_field() != "attributedTo":
raise
value = getattr(data, "actor")
- formatted = self.field_from_activity(value)
+ formatted = self.field_from_activity(
+ value, allow_external_connections=allow_external_connections
+ )
if formatted is None or formatted is MISSING or formatted == {}:
return False
@@ -115,7 +122,8 @@ class ActivitypubFieldMixin:
return {self.activitypub_wrapper: value}
return value
- def field_from_activity(self, value):
+ # pylint: disable=unused-argument
+ def field_from_activity(self, value, allow_external_connections=True):
"""formatter to convert activitypub into a model value"""
if value and hasattr(self, "activitypub_wrapper"):
value = value.get(self.activitypub_wrapper)
@@ -137,7 +145,7 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
self.load_remote = load_remote
super().__init__(*args, **kwargs)
- def field_from_activity(self, value):
+ def field_from_activity(self, value, allow_external_connections=True):
if not value:
return None
@@ -158,7 +166,11 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
if not self.load_remote:
# only look in the local database
return related_model.find_existing_by_remote_id(value)
- return activitypub.resolve_remote_id(value, model=related_model)
+ return activitypub.resolve_remote_id(
+ value,
+ model=related_model,
+ allow_external_connections=allow_external_connections,
+ )
class RemoteIdField(ActivitypubFieldMixin, models.CharField):
@@ -210,7 +222,7 @@ PrivacyLevels = [
class PrivacyField(ActivitypubFieldMixin, models.CharField):
- """this maps to two differente activitypub fields"""
+ """this maps to two different activitypub fields"""
public = "https://www.w3.org/ns/activitystreams#Public"
@@ -218,7 +230,9 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
super().__init__(*args, max_length=255, choices=PrivacyLevels, default="public")
# pylint: disable=invalid-name
- def set_field_from_activity(self, instance, data, overwrite=True):
+ def set_field_from_activity(
+ self, instance, data, overwrite=True, allow_external_connections=True
+ ):
if not overwrite:
return False
@@ -233,7 +247,11 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
break
if not user_field:
raise ValidationError("No user field found for privacy", data)
- user = activitypub.resolve_remote_id(getattr(data, user_field), model="User")
+ user = activitypub.resolve_remote_id(
+ getattr(data, user_field),
+ model="User",
+ allow_external_connections=allow_external_connections,
+ )
if to == [self.public]:
setattr(instance, self.name, "public")
@@ -294,13 +312,17 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
self.link_only = link_only
super().__init__(*args, **kwargs)
- def set_field_from_activity(self, instance, data, overwrite=True):
+ def set_field_from_activity(
+ self, instance, data, overwrite=True, allow_external_connections=True
+ ):
"""helper function for assigning a value to the field"""
if not overwrite and getattr(instance, self.name).exists():
return False
value = getattr(data, self.get_activitypub_field())
- formatted = self.field_from_activity(value)
+ formatted = self.field_from_activity(
+ value, allow_external_connections=allow_external_connections
+ )
if formatted is None or formatted is MISSING:
return False
getattr(instance, self.name).set(formatted)
@@ -312,7 +334,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
return f"{value.instance.remote_id}/{self.name}"
return [i.remote_id for i in value.all()]
- def field_from_activity(self, value):
+ def field_from_activity(self, value, allow_external_connections=True):
if value is None or value is MISSING:
return None
if not isinstance(value, list):
@@ -325,7 +347,11 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
except ValidationError:
continue
items.append(
- activitypub.resolve_remote_id(remote_id, model=self.related_model)
+ activitypub.resolve_remote_id(
+ remote_id,
+ model=self.related_model,
+ allow_external_connections=allow_external_connections,
+ )
)
return items
@@ -343,18 +369,29 @@ class TagField(ManyToManyField):
activity_type = item.__class__.__name__
if activity_type == "User":
activity_type = "Mention"
+
+ if activity_type == "Hashtag":
+ name = item.name
+ else:
+ name = f"@{getattr(item, item.name_field)}"
+
tags.append(
activitypub.Link(
href=item.remote_id,
- name=getattr(item, item.name_field),
+ name=name,
type=activity_type,
)
)
return tags
- def field_from_activity(self, value):
+ def field_from_activity(self, value, allow_external_connections=True):
if not isinstance(value, list):
- return None
+ # GoToSocial DMs and single-user mentions are
+ # sent as objects, not as an array of objects
+ if isinstance(value, dict):
+ value = [value]
+ else:
+ return None
items = []
for link_json in value:
link = activitypub.Link(**link_json)
@@ -364,9 +401,22 @@ class TagField(ManyToManyField):
if tag_type != self.related_model.activity_serializer.type:
# tags can contain multiple types
continue
- items.append(
- activitypub.resolve_remote_id(link.href, model=self.related_model)
- )
+
+ if tag_type == "Hashtag":
+ # we already have all data to create hashtags,
+ # no need to fetch from remote
+ item = self.related_model.activity_serializer(**link_json)
+ hashtag = item.to_model(model=self.related_model, save=True)
+ items.append(hashtag)
+ else:
+ # for other tag types we fetch them remotely
+ items.append(
+ activitypub.resolve_remote_id(
+ link.href,
+ model=self.related_model,
+ allow_external_connections=allow_external_connections,
+ )
+ )
return items
@@ -389,11 +439,15 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
self.alt_field = alt_field
super().__init__(*args, **kwargs)
- # pylint: disable=arguments-differ,arguments-renamed
- def set_field_from_activity(self, instance, data, save=True, overwrite=True):
- """helper function for assinging a value to the field"""
+ # pylint: disable=arguments-differ,arguments-renamed,too-many-arguments
+ def set_field_from_activity(
+ self, instance, data, save=True, overwrite=True, allow_external_connections=True
+ ):
+ """helper function for assigning a value to the field"""
value = getattr(data, self.get_activitypub_field())
- formatted = self.field_from_activity(value)
+ formatted = self.field_from_activity(
+ value, allow_external_connections=allow_external_connections
+ )
if formatted is None or formatted is MISSING:
return False
@@ -425,7 +479,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
return activitypub.Document(url=url, name=alt)
- def field_from_activity(self, value):
+ def field_from_activity(self, value, allow_external_connections=True):
image_slug = value
# when it's an inline image (User avatar/icon, Book cover), it's a json
# blob, but when it's an attached image, it's just a url
@@ -480,9 +534,11 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
return None
return value.isoformat()
- def field_from_activity(self, value):
+ def field_from_activity(self, value, allow_external_connections=True):
+ missing_fields = datetime(1970, 1, 1) # "2022-10" => "2022-10-01"
try:
- date_value = dateutil.parser.parse(value)
+ # TODO(dato): investigate `ignoretz=True` wrt bookwyrm#3028.
+ date_value = dateutil.parser.parse(value, default=missing_fields)
try:
return timezone.make_aware(date_value)
except ValueError:
@@ -494,11 +550,14 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
class HtmlField(ActivitypubFieldMixin, models.TextField):
"""a text field for storing html"""
- def field_from_activity(self, value):
+ def field_from_activity(self, value, allow_external_connections=True):
if not value or value == MISSING:
return None
return clean(value)
+ def field_to_activity(self, value):
+ return markdown(value) if value else value
+
class ArrayField(ActivitypubFieldMixin, DjangoArrayField):
"""activitypub-aware array field"""
@@ -511,6 +570,10 @@ class CharField(ActivitypubFieldMixin, models.CharField):
"""activitypub-aware char field"""
+class CICharField(ActivitypubFieldMixin, DjangoCICharField):
+ """activitypub-aware cichar field"""
+
+
class URLField(ActivitypubFieldMixin, models.URLField):
"""activitypub-aware url field"""
diff --git a/bookwyrm/models/hashtag.py b/bookwyrm/models/hashtag.py
new file mode 100644
index 000000000..7894a3528
--- /dev/null
+++ b/bookwyrm/models/hashtag.py
@@ -0,0 +1,23 @@
+""" model for tags """
+from bookwyrm import activitypub
+from .activitypub_mixin import ActivitypubMixin
+from .base_model import BookWyrmModel
+from .fields import CICharField
+
+
+class Hashtag(ActivitypubMixin, BookWyrmModel):
+ "a hashtag which can be used in statuses"
+
+ name = CICharField(
+ max_length=256,
+ blank=False,
+ null=False,
+ activitypub_field="name",
+ deduplication_field=True,
+ )
+
+ name_field = "name"
+ activity_serializer = activitypub.Hashtag
+
+ def __repr__(self):
+ return f"<{self.__class__} id={self.id} name={self.name}>"
diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py
index d8cfad314..f5d86ad2e 100644
--- a/bookwyrm/models/import_job.py
+++ b/bookwyrm/models/import_job.py
@@ -1,4 +1,5 @@
""" track progress of goodreads imports """
+from datetime import datetime
import math
import re
import dateutil.parser
@@ -19,7 +20,7 @@ from bookwyrm.models import (
Review,
ReviewRating,
)
-from bookwyrm.tasks import app, LOW
+from bookwyrm.tasks import app, IMPORT_TRIGGERED, IMPORTS
from .fields import PrivacyLevels
@@ -54,10 +55,10 @@ ImportStatuses = [
class ImportJob(models.Model):
"""entry for a specific request for book data import"""
- user = models.ForeignKey(User, on_delete=models.CASCADE)
+ user: User = models.ForeignKey(User, on_delete=models.CASCADE)
created_date = models.DateTimeField(default=timezone.now)
updated_date = models.DateTimeField(default=timezone.now)
- include_reviews = models.BooleanField(default=True)
+ include_reviews: bool = models.BooleanField(default=True)
mappings = models.JSONField()
source = models.CharField(max_length=100)
privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels)
@@ -74,10 +75,9 @@ class ImportJob(models.Model):
task = start_import_task.delay(self.id)
self.task_id = task.id
- self.status = "active"
- self.save(update_fields=["status", "task_id"])
+ self.save(update_fields=["task_id"])
- def complete_job(self):
+ def complete_job(self) -> None:
"""Report that the job has completed"""
self.status = "complete"
self.complete = True
@@ -253,42 +253,37 @@ class ImportItem(models.Model):
@property
def rating(self):
"""x/5 star rating for a book"""
- if self.normalized_data.get("rating"):
+ if not self.normalized_data.get("rating"):
+ return None
+ try:
return float(self.normalized_data.get("rating"))
- return None
+ except ValueError:
+ return None
+
+ def _parse_datefield(self, field, /):
+ if not (date := self.normalized_data.get(field)):
+ return None
+
+ defaults = datetime(1970, 1, 1) # "2022-10" => "2022-10-01"
+ parsed = dateutil.parser.parse(date, default=defaults)
+
+ # Keep timezone if import already had one, else use default.
+ return parsed if timezone.is_aware(parsed) else timezone.make_aware(parsed)
@property
def date_added(self):
"""when the book was added to this dataset"""
- if self.normalized_data.get("date_added"):
- parsed_date_added = dateutil.parser.parse(
- self.normalized_data.get("date_added")
- )
-
- if timezone.is_aware(parsed_date_added):
- # Keep timezone if import already had one
- return parsed_date_added
-
- return timezone.make_aware(parsed_date_added)
- return None
+ return self._parse_datefield("date_added")
@property
def date_started(self):
"""when the book was started"""
- if self.normalized_data.get("date_started"):
- return timezone.make_aware(
- dateutil.parser.parse(self.normalized_data.get("date_started"))
- )
- return None
+ return self._parse_datefield("date_started")
@property
def date_read(self):
"""the date a book was completed"""
- if self.normalized_data.get("date_finished"):
- return timezone.make_aware(
- dateutil.parser.parse(self.normalized_data.get("date_finished"))
- )
- return None
+ return self._parse_datefield("date_finished")
@property
def reads(self):
@@ -328,10 +323,12 @@ class ImportItem(models.Model):
)
-@app.task(queue=LOW)
+@app.task(queue=IMPORTS)
def start_import_task(job_id):
"""trigger the child tasks for each row"""
job = ImportJob.objects.get(id=job_id)
+ job.status = "active"
+ job.save(update_fields=["status"])
# don't start the job if it was stopped from the UI
if job.complete:
return
@@ -345,7 +342,7 @@ def start_import_task(job_id):
job.save()
-@app.task(queue=LOW)
+@app.task(queue=IMPORTS)
def import_item_task(item_id):
"""resolve a row into a book"""
item = ImportItem.objects.get(id=item_id)
@@ -395,7 +392,7 @@ def handle_imported_book(item):
shelved_date = item.date_added or timezone.now()
ShelfBook(
book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date
- ).save(priority=LOW)
+ ).save(priority=IMPORT_TRIGGERED)
for read in item.reads:
# check for an existing readthrough with the same dates
@@ -437,7 +434,7 @@ def handle_imported_book(item):
published_date=published_date_guess,
privacy=job.privacy,
)
- review.save(software="bookwyrm", priority=LOW)
+ review.save(software="bookwyrm", priority=IMPORT_TRIGGERED)
else:
# just a rating
review = ReviewRating.objects.filter(
@@ -454,7 +451,7 @@ def handle_imported_book(item):
published_date=published_date_guess,
privacy=job.privacy,
)
- review.save(software="bookwyrm", priority=LOW)
+ review.save(software="bookwyrm", priority=IMPORT_TRIGGERED)
# only broadcast this review to other bookwyrm instances
item.linked_review = review
diff --git a/bookwyrm/models/link.py b/bookwyrm/models/link.py
index 56b096bc2..d334a9d29 100644
--- a/bookwyrm/models/link.py
+++ b/bookwyrm/models/link.py
@@ -31,7 +31,7 @@ class Link(ActivitypubMixin, BookWyrmModel):
@property
def name(self):
- """link name via the assocaited domain"""
+ """link name via the associated domain"""
return self.domain.name
def save(self, *args, **kwargs):
diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py
index fa2ce54e2..522038f9a 100644
--- a/bookwyrm/models/notification.py
+++ b/bookwyrm/models/notification.py
@@ -2,8 +2,8 @@
from django.db import models, transaction
from django.dispatch import receiver
from .base_model import BookWyrmModel
-from . import Boost, Favorite, GroupMemberInvitation, ImportJob, ListItem, Report
-from . import Status, User, UserFollowRequest
+from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain
+from . import ListItem, Report, Status, User, UserFollowRequest
class Notification(BookWyrmModel):
@@ -28,6 +28,7 @@ class Notification(BookWyrmModel):
# Admin
REPORT = "REPORT"
+ LINK_DOMAIN = "LINK_DOMAIN"
# Groups
INVITE = "INVITE"
@@ -43,7 +44,7 @@ class Notification(BookWyrmModel):
NotificationType = models.TextChoices(
# there has got be a better way to do this
"NotificationType",
- f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
+ f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
)
user = models.ForeignKey("User", on_delete=models.CASCADE)
@@ -64,6 +65,7 @@ class Notification(BookWyrmModel):
"ListItem", symmetrical=False, related_name="notifications"
)
related_reports = models.ManyToManyField("Report", symmetrical=False)
+ related_link_domains = models.ManyToManyField("LinkDomain", symmetrical=False)
@classmethod
@transaction.atomic
@@ -241,6 +243,26 @@ def notify_admins_on_report(sender, instance, created, *args, **kwargs):
notification.related_reports.add(instance)
+@receiver(models.signals.post_save, sender=LinkDomain)
+@transaction.atomic
+# pylint: disable=unused-argument
+def notify_admins_on_link_domain(sender, instance, created, *args, **kwargs):
+ """a new link domain needs to be verified"""
+ if not created:
+ # otherwise you'll get a notification when you approve a domain
+ return
+
+ # moderators and superusers should be notified
+ admins = User.admins()
+ for admin in admins:
+ notification, _ = Notification.objects.get_or_create(
+ user=admin,
+ notification_type=Notification.LINK_DOMAIN,
+ read=False,
+ )
+ notification.related_link_domains.add(instance)
+
+
@receiver(models.signals.post_save, sender=GroupMemberInvitation)
# pylint: disable=unused-argument
def notify_user_on_group_invite(sender, instance, *args, **kwargs):
@@ -262,7 +284,7 @@ def notify_user_on_list_item_add(sender, instance, created, *args, **kwargs):
return
list_owner = instance.book_list.user
- # create a notification if somoene ELSE added to a local user's list
+ # create a notification if someone ELSE added to a local user's list
if list_owner.local and list_owner != instance.user:
# keep the related_user singular, group the items
Notification.notify_list_item(list_owner, instance)
diff --git a/bookwyrm/models/readthrough.py b/bookwyrm/models/readthrough.py
index 314b40a5c..4911c715b 100644
--- a/bookwyrm/models/readthrough.py
+++ b/bookwyrm/models/readthrough.py
@@ -8,7 +8,7 @@ from .base_model import BookWyrmModel
class ProgressMode(models.TextChoices):
- """types of prgress available"""
+ """types of progress available"""
PAGE = "PG", "page"
PERCENT = "PCT", "percent"
@@ -32,7 +32,7 @@ class ReadThrough(BookWyrmModel):
def save(self, *args, **kwargs):
"""update user active time"""
- cache.delete(f"latest_read_through-{self.user.id}-{self.book.id}")
+ cache.delete(f"latest_read_through-{self.user_id}-{self.book_id}")
self.user.update_active_date()
# an active readthrough must have an unset finish date
if self.finish_date or self.stopped_date:
diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py
index 082294c0e..7af6ad5ab 100644
--- a/bookwyrm/models/relationship.py
+++ b/bookwyrm/models/relationship.py
@@ -33,7 +33,7 @@ class UserRelationship(BookWyrmModel):
@property
def recipients(self):
- """the remote user needs to recieve direct broadcasts"""
+ """the remote user needs to receive direct broadcasts"""
return [u for u in [self.user_subject, self.user_object] if not u.local]
def save(self, *args, **kwargs):
@@ -139,6 +139,7 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
)
super().save(*args, **kwargs)
+ # a local user is following a remote user
if broadcast and self.user_subject.local and not self.user_object.local:
self.broadcast(self.to_activity(), self.user_subject)
@@ -157,6 +158,7 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
def accept(self, broadcast_only=False):
"""turn this request into the real deal"""
user = self.user_object
+ # broadcast when accepting a remote request
if not self.user_subject.local:
activity = activitypub.Accept(
id=self.get_accept_reject_id(status="accepts"),
@@ -168,7 +170,11 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
return
with transaction.atomic():
- UserFollows.from_request(self)
+ try:
+ UserFollows.from_request(self)
+ except IntegrityError:
+ # this just means we already saved this relationship
+ pass
if self.id:
self.delete()
diff --git a/bookwyrm/models/report.py b/bookwyrm/models/report.py
index f6e665053..74a9bbe41 100644
--- a/bookwyrm/models/report.py
+++ b/bookwyrm/models/report.py
@@ -1,11 +1,27 @@
""" flagged for moderation """
from django.core.exceptions import PermissionDenied
from django.db import models
+from django.utils.translation import gettext_lazy as _
from bookwyrm.settings import DOMAIN
from .base_model import BookWyrmModel
+# Report action enums
+COMMENT = "comment"
+RESOLVE = "resolve"
+REOPEN = "reopen"
+MESSAGE_REPORTER = "message_reporter"
+MESSAGE_OFFENDER = "message_offender"
+USER_SUSPENSION = "user_suspension"
+USER_UNSUSPENSION = "user_unsuspension"
+USER_DELETION = "user_deletion"
+USER_PERMS = "user_perms"
+BLOCK_DOMAIN = "block_domain"
+APPROVE_DOMAIN = "approve_domain"
+DELETE_ITEM = "delete_item"
+
+
class Report(BookWyrmModel):
"""reported status or user"""
@@ -32,20 +48,65 @@ class Report(BookWyrmModel):
def get_remote_id(self):
return f"https://{DOMAIN}/settings/reports/{self.id}"
+ def comment(self, user, note):
+ """comment on a report"""
+ ReportAction.objects.create(
+ action_type=COMMENT, user=user, note=note, report=self
+ )
+
+ def resolve(self, user):
+ """Mark a report as complete"""
+ self.resolved = True
+ self.save()
+ ReportAction.objects.create(action_type=RESOLVE, user=user, report=self)
+
+ def reopen(self, user):
+ """Wait! This report isn't complete after all"""
+ self.resolved = False
+ self.save()
+ ReportAction.objects.create(action_type=REOPEN, user=user, report=self)
+
+ @classmethod
+ def record_action(cls, report_id: int, action: str, user):
+ """Note that someone did something"""
+ if not report_id:
+ return
+ report = cls.objects.get(id=report_id)
+ ReportAction.objects.create(action_type=action, user=user, report=report)
+
class Meta:
"""set order by default"""
ordering = ("-created_date",)
-class ReportComment(BookWyrmModel):
+ReportActionTypes = [
+ (COMMENT, _("Comment")),
+ (RESOLVE, _("Resolved report")),
+ (REOPEN, _("Re-opened report")),
+ (MESSAGE_REPORTER, _("Messaged reporter")),
+ (MESSAGE_OFFENDER, _("Messaged reported user")),
+ (USER_SUSPENSION, _("Suspended user")),
+ (USER_UNSUSPENSION, _("Un-suspended user")),
+ (USER_PERMS, _("Changed user permission level")),
+ (USER_DELETION, _("Deleted user account")),
+ (BLOCK_DOMAIN, _("Blocked domain")),
+ (APPROVE_DOMAIN, _("Approved domain")),
+ (DELETE_ITEM, _("Deleted item")),
+]
+
+
+class ReportAction(BookWyrmModel):
"""updates on a report"""
user = models.ForeignKey("User", on_delete=models.PROTECT)
+ action_type = models.CharField(
+ max_length=20, blank=False, default="comment", choices=ReportActionTypes
+ )
note = models.TextField()
report = models.ForeignKey(Report, on_delete=models.PROTECT)
class Meta:
"""sort comments"""
- ordering = ("-created_date",)
+ ordering = ("created_date",)
diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py
index d955e8d07..3d92f8d43 100644
--- a/bookwyrm/models/shelf.py
+++ b/bookwyrm/models/shelf.py
@@ -7,6 +7,7 @@ from django.utils import timezone
from bookwyrm import activitypub
from bookwyrm.settings import DOMAIN
+from bookwyrm.tasks import BROADCAST
from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
from .base_model import BookWyrmModel
from . import fields
@@ -39,9 +40,9 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
activity_serializer = activitypub.Shelf
- def save(self, *args, **kwargs):
+ def save(self, *args, priority=BROADCAST, **kwargs):
"""set the identifier"""
- super().save(*args, **kwargs)
+ super().save(*args, priority=priority, **kwargs)
if not self.identifier:
self.identifier = self.get_identifier()
super().save(*args, **kwargs, broadcast=False)
@@ -79,7 +80,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel):
raise PermissionDenied()
class Meta:
- """user/shelf unqiueness"""
+ """user/shelf uniqueness"""
unique_together = ("user", "identifier")
@@ -99,24 +100,24 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel):
activity_serializer = activitypub.ShelfItem
collection_field = "shelf"
- def save(self, *args, **kwargs):
+ def save(self, *args, priority=BROADCAST, **kwargs):
if not self.user:
self.user = self.shelf.user
if self.id and self.user.local:
# remove all caches related to all editions of this book
cache.delete_many(
[
- f"book-on-shelf-{book.id}-{self.shelf.id}"
+ f"book-on-shelf-{book.id}-{self.shelf_id}"
for book in self.book.parent_work.editions.all()
]
)
- super().save(*args, **kwargs)
+ super().save(*args, priority=priority, **kwargs)
def delete(self, *args, **kwargs):
if self.id and self.user.local:
cache.delete_many(
[
- f"book-on-shelf-{book}-{self.shelf.id}"
+ f"book-on-shelf-{book}-{self.shelf_id}"
for book in self.book.parent_work.editions.values_list(
"id", flat=True
)
diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py
index 0f56162b1..a27c4b70d 100644
--- a/bookwyrm/models/site.py
+++ b/bookwyrm/models/site.py
@@ -3,6 +3,7 @@ import datetime
from urllib.parse import urljoin
import uuid
+import django.contrib.auth.models as auth_models
from django.core.exceptions import PermissionDenied
from django.db import models, IntegrityError
from django.dispatch import receiver
@@ -62,12 +63,17 @@ class SiteSettings(SiteModel):
)
code_of_conduct = models.TextField(default="Add a code of conduct here.")
privacy_policy = models.TextField(default="Add a privacy policy here.")
+ impressum = models.TextField(default="Add a impressum here.")
+ show_impressum = models.BooleanField(default=False)
# registration
allow_registration = models.BooleanField(default=False)
allow_invite_requests = models.BooleanField(default=True)
invite_request_question = models.BooleanField(default=False)
require_confirm_email = models.BooleanField(default=True)
+ default_user_auth_group = models.ForeignKey(
+ auth_models.Group, null=True, blank=True, on_delete=models.RESTRICT
+ )
invite_question_text = models.CharField(
max_length=255, blank=True, default="What is your favourite book?"
@@ -86,6 +92,11 @@ class SiteSettings(SiteModel):
admin_email = models.EmailField(max_length=255, null=True, blank=True)
footer_item = models.TextField(null=True, blank=True)
+ # controls
+ imports_enabled = models.BooleanField(default=True)
+ import_size_limit = models.IntegerField(default=0)
+ import_limit_reset = models.IntegerField(default=0)
+
field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])
@classmethod
@@ -181,7 +192,7 @@ class InviteRequest(BookWyrmModel):
invite = models.ForeignKey(
SiteInvite, on_delete=models.SET_NULL, null=True, blank=True
)
- answer = models.TextField(max_length=50, unique=False, null=True, blank=True)
+ answer = models.TextField(max_length=255, unique=False, null=True, blank=True)
invite_sent = models.BooleanField(default=False)
ignored = models.BooleanField(default=False)
@@ -198,7 +209,7 @@ class InviteRequest(BookWyrmModel):
super().save(*args, **kwargs)
-def get_passowrd_reset_expiry():
+def get_password_reset_expiry():
"""give people a limited time to use the link"""
now = timezone.now()
return now + datetime.timedelta(days=1)
@@ -208,7 +219,7 @@ class PasswordReset(models.Model):
"""gives someone access to create an account on the instance"""
code = models.CharField(max_length=32, default=new_access_code)
- expiry = models.DateTimeField(default=get_passowrd_reset_expiry)
+ expiry = models.DateTimeField(default=get_password_reset_expiry)
user = models.OneToOneField(User, on_delete=models.CASCADE)
def valid(self):
diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py
index 1546a4b5b..5d6109468 100644
--- a/bookwyrm/models/status.py
+++ b/bookwyrm/models/status.py
@@ -1,5 +1,6 @@
""" models for storing different kinds of Activities """
from dataclasses import MISSING
+from typing import Optional
import re
from django.apps import apps
@@ -34,6 +35,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
raw_content = models.TextField(blank=True, null=True)
mention_users = fields.TagField("User", related_name="mention_user")
mention_books = fields.TagField("Edition", related_name="mention_book")
+ mention_hashtags = fields.TagField("Hashtag", related_name="mention_hashtag")
local = models.BooleanField(default=True)
content_warning = fields.CharField(
max_length=500, blank=True, null=True, activitypub_field="summary"
@@ -63,6 +65,9 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
activitypub_field="inReplyTo",
)
thread_id = models.IntegerField(blank=True, null=True)
+ # statuses get saved a few times, this indicates if they're set
+ ready = models.BooleanField(default=True)
+
objects = InheritanceManager()
activity_serializer = activitypub.Note
@@ -77,14 +82,13 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
def save(self, *args, **kwargs):
"""save and notify"""
if self.reply_parent:
- self.thread_id = self.reply_parent.thread_id or self.reply_parent.id
+ self.thread_id = self.reply_parent.thread_id or self.reply_parent_id
super().save(*args, **kwargs)
if not self.reply_parent:
self.thread_id = self.id
-
- super().save(broadcast=False, update_fields=["thread_id"])
+ super().save(broadcast=False, update_fields=["thread_id"])
def delete(self, *args, **kwargs): # pylint: disable=unused-argument
""" "delete" a status"""
@@ -113,10 +117,16 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
return list(set(mentions))
@classmethod
- def ignore_activity(cls, activity): # pylint: disable=too-many-return-statements
+ def ignore_activity(
+ cls, activity, allow_external_connections=True
+ ): # pylint: disable=too-many-return-statements
"""keep notes if they are replies to existing statuses"""
if activity.type == "Announce":
- boosted = activitypub.resolve_remote_id(activity.object, get_activity=True)
+ boosted = activitypub.resolve_remote_id(
+ activity.object,
+ get_activity=True,
+ allow_external_connections=allow_external_connections,
+ )
if not boosted:
# if we can't load the status, definitely ignore it
return True
@@ -133,10 +143,17 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
# keep notes if they mention local users
if activity.tag == MISSING or activity.tag is None:
return True
- tags = [l["href"] for l in activity.tag if l["type"] == "Mention"]
+ # GoToSocial sends single tags as objects
+ # not wrapped in a list
+ tags = activity.tag if isinstance(activity.tag, list) else [activity.tag]
user_model = apps.get_model("bookwyrm.User", require_ready=True)
for tag in tags:
- if user_model.objects.filter(remote_id=tag, local=True).exists():
+ if (
+ tag["type"] == "Mention"
+ and user_model.objects.filter(
+ remote_id=tag["href"], local=True
+ ).exists()
+ ):
# we found a mention of a known use boost
return False
return True
@@ -253,7 +270,7 @@ class GeneratedNote(Status):
"""indicate the book in question for mastodon (or w/e) users"""
message = self.content
books = ", ".join(
- f'"{book.title}"'
+ f'{book.title}'
for book in self.mention_books.all()
)
return f"{self.user.display_name} {message} {books}"
@@ -304,17 +321,14 @@ class Comment(BookStatus):
@property
def pure_content(self):
"""indicate the book in question for mastodon (or w/e) users"""
- if self.progress_mode == "PG" and self.progress and (self.progress > 0):
- return_value = (
- f'{self.content}(comment on '
- f'"{self.book.title}", page {self.progress})
'
- )
- else:
- return_value = (
- f'{self.content}(comment on '
- f'"{self.book.title}")
'
- )
- return return_value
+ progress = self.progress or 0
+ citation = (
+ f'comment on '
+ f"{self.book.title}"
+ )
+ if self.progress_mode == "PG" and progress > 0:
+ citation += f", p. {progress}"
+ return f"{self.content}({citation})
"
activity_serializer = activitypub.Comment
@@ -327,6 +341,9 @@ class Quotation(BookStatus):
position = models.IntegerField(
validators=[MinValueValidator(0)], null=True, blank=True
)
+ endposition = models.IntegerField(
+ validators=[MinValueValidator(0)], null=True, blank=True
+ )
position_mode = models.CharField(
max_length=3,
choices=ProgressMode.choices,
@@ -335,22 +352,24 @@ class Quotation(BookStatus):
blank=True,
)
+ def _format_position(self) -> Optional[str]:
+ """serialize page position"""
+ beg = self.position
+ end = self.endposition or 0
+ if self.position_mode != "PG" or not beg:
+ return None
+ return f"pp. {beg}-{end}" if end > beg else f"p. {beg}"
+
@property
def pure_content(self):
"""indicate the book in question for mastodon (or w/e) users"""
quote = re.sub(r"^", '
"', self.quote)
quote = re.sub(r"
$", '"', quote)
- if self.position_mode == "PG" and self.position and (self.position > 0):
- return_value = (
- f'{quote} -- '
- f'"{self.book.title}", page {self.position}
{self.content}'
- )
- else:
- return_value = (
- f'{quote} -- '
- f'"{self.book.title}"
{self.content}'
- )
- return return_value
+ title, href = self.book.title, self.book.remote_id
+ citation = f'— {title}'
+ if position := self._format_position():
+ citation += f", {position}"
+ return f"{quote} {citation}
{self.content}"
activity_serializer = activitypub.Quotation
@@ -399,7 +418,7 @@ class ReviewRating(Review):
def save(self, *args, **kwargs):
if not self.rating:
raise ValueError("ReviewRating object must include a numerical rating")
- return super().save(*args, **kwargs)
+ super().save(*args, **kwargs)
@property
def pure_content(self):
diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py
index 5f7b00d87..6e0912aec 100644
--- a/bookwyrm/models/user.py
+++ b/bookwyrm/models/user.py
@@ -3,9 +3,9 @@ import re
from urllib.parse import urlparse
from django.apps import apps
-from django.contrib.auth.models import AbstractUser, Group
+from django.contrib.auth.models import AbstractUser
from django.contrib.postgres.fields import ArrayField, CICharField
-from django.core.exceptions import PermissionDenied
+from django.core.exceptions import PermissionDenied, ObjectDoesNotExist
from django.dispatch import receiver
from django.db import models, transaction
from django.utils import timezone
@@ -20,7 +20,7 @@ from bookwyrm.models.status import Status
from bookwyrm.preview_images import generate_user_preview_image_task
from bookwyrm.settings import DOMAIN, ENABLE_PREVIEW_IMAGES, USE_HTTPS, LANGUAGES
from bookwyrm.signatures import create_key_pair
-from bookwyrm.tasks import app, LOW
+from bookwyrm.tasks import app, MISC
from bookwyrm.utils import regex
from .activitypub_mixin import OrderedCollectionPageMixin, ActivitypubMixin
from .base_model import BookWyrmModel, DeactivationReason, new_access_code
@@ -244,9 +244,10 @@ class User(OrderedCollectionPageMixin, AbstractUser):
def admins(cls):
"""Get a queryset of the admins for this instance"""
return cls.objects.filter(
- models.Q(user_permissions__name__in=["moderate_user", "moderate_post"])
- | models.Q(is_superuser=True)
- )
+ models.Q(groups__name__in=["moderator", "admin"])
+ | models.Q(is_superuser=True),
+ is_active=True,
+ ).distinct()
def update_active_date(self):
"""this user is here! they are doing things!"""
@@ -338,7 +339,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
# this is a new remote user, we need to set their remote server field
if not self.local:
super().save(*args, **kwargs)
- transaction.on_commit(lambda: set_remote_server.delay(self.id))
+ transaction.on_commit(lambda: set_remote_server(self.id))
return
with transaction.atomic():
@@ -355,8 +356,14 @@ class User(OrderedCollectionPageMixin, AbstractUser):
# make users editors by default
try:
- self.groups.add(Group.objects.get(name="editor"))
- except Group.DoesNotExist:
+ group = (
+ apps.get_model("bookwyrm.SiteSettings")
+ .objects.get()
+ .default_user_auth_group
+ )
+ if group:
+ self.groups.add(group)
+ except ObjectDoesNotExist:
# this should only happen in tests
pass
@@ -372,6 +379,7 @@ class User(OrderedCollectionPageMixin, AbstractUser):
"""We don't actually delete the database entry"""
# pylint: disable=attribute-defined-outside-init
self.is_active = False
+ self.avatar = ""
# skip the logic in this class's save()
super().save(*args, **kwargs)
@@ -386,10 +394,15 @@ class User(OrderedCollectionPageMixin, AbstractUser):
def reactivate(self):
"""Now you want to come back, huh?"""
# pylint: disable=attribute-defined-outside-init
+ if not self.allow_reactivation:
+ return
self.is_active = True
self.deactivation_reason = None
self.allow_reactivation = False
- super().save(broadcast=False)
+ super().save(
+ broadcast=False,
+ update_fields=["deactivation_reason", "is_active", "allow_reactivation"],
+ )
@property
def local_path(self):
@@ -458,18 +471,30 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
return super().save(*args, **kwargs)
-@app.task(queue=LOW)
-def set_remote_server(user_id):
+@app.task(queue=MISC)
+def set_remote_server(user_id, allow_external_connections=False):
"""figure out the user's remote server in the background"""
user = User.objects.get(id=user_id)
actor_parts = urlparse(user.remote_id)
- user.federated_server = get_or_create_remote_server(actor_parts.netloc)
+ federated_server = get_or_create_remote_server(
+ actor_parts.netloc, allow_external_connections=allow_external_connections
+ )
+ # if we were unable to find the server, we need to create a new entry for it
+ if not federated_server:
+ # and to do that, we will call this function asynchronously.
+ if not allow_external_connections:
+ set_remote_server.delay(user_id, allow_external_connections=True)
+ return
+
+ user.federated_server = federated_server
user.save(broadcast=False, update_fields=["federated_server"])
if user.bookwyrm_user and user.outbox:
get_remote_reviews.delay(user.outbox)
-def get_or_create_remote_server(domain, refresh=False):
+def get_or_create_remote_server(
+ domain, allow_external_connections=False, refresh=False
+):
"""get info on a remote server"""
server = FederatedServer()
try:
@@ -479,6 +504,9 @@ def get_or_create_remote_server(domain, refresh=False):
except FederatedServer.DoesNotExist:
pass
+ if not allow_external_connections:
+ return None
+
try:
data = get_data(f"https://{domain}/.well-known/nodeinfo")
try:
@@ -502,7 +530,7 @@ def get_or_create_remote_server(domain, refresh=False):
return server
-@app.task(queue=LOW)
+@app.task(queue=MISC)
def get_remote_reviews(outbox):
"""ingest reviews by a new remote bookwyrm user"""
outbox_page = outbox + "?page=true&type=Review"
@@ -521,6 +549,11 @@ def preview_image(instance, *args, **kwargs):
"""create preview images when user is updated"""
if not ENABLE_PREVIEW_IMAGES:
return
+
+ # don't call the task for remote users
+ if not instance.local:
+ return
+
changed_fields = instance.field_tracker.changed()
if len(changed_fields) > 0:
diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py
index d20145cd3..aba372abc 100644
--- a/bookwyrm/preview_images.py
+++ b/bookwyrm/preview_images.py
@@ -16,7 +16,7 @@ from django.core.files.storage import default_storage
from django.db.models import Avg
from bookwyrm import models, settings
-from bookwyrm.tasks import app, LOW
+from bookwyrm.tasks import app, IMAGES
logger = logging.getLogger(__name__)
@@ -71,20 +71,29 @@ def get_wrapped_text(text, font, content_width):
low = 0
high = len(text)
+ draw = ImageDraw.Draw(Image.new("RGB", (100, 100)))
+
try:
# ideal length is determined via binary search
while low < high:
mid = math.floor(low + high)
wrapped_text = textwrap.fill(text, width=mid)
- width = font.getsize_multiline(wrapped_text)[0]
+
+ left, top, right, bottom = draw.multiline_textbbox(
+ (0, 0), wrapped_text, font=font
+ )
+ width = right - left
+ height = bottom - top
+
if width < content_width:
low = mid
else:
high = mid - 1
except AttributeError:
wrapped_text = text
+ height = 26
- return wrapped_text
+ return wrapped_text, height
def generate_texts_layer(texts, content_width):
@@ -100,47 +109,53 @@ def generate_texts_layer(texts, content_width):
text_y = 0
if "text_zero" in texts and texts["text_zero"]:
- # Text one (Book title)
- text_zero = get_wrapped_text(texts["text_zero"], font_text_zero, content_width)
+ # Text zero (Site preview domain name)
+ text_zero, text_height = get_wrapped_text(
+ texts["text_zero"], font_text_zero, content_width
+ )
text_layer_draw.multiline_text(
(0, text_y), text_zero, font=font_text_zero, fill=TEXT_COLOR
)
try:
- text_y = text_y + font_text_zero.getsize_multiline(text_zero)[1] + 16
+ text_y = text_y + text_height + 16
except (AttributeError, IndexError):
text_y = text_y + 26
if "text_one" in texts and texts["text_one"]:
- # Text one (Book title)
- text_one = get_wrapped_text(texts["text_one"], font_text_one, content_width)
+ # Text one (Book/Site title, User display name)
+ text_one, text_height = get_wrapped_text(
+ texts["text_one"], font_text_one, content_width
+ )
text_layer_draw.multiline_text(
(0, text_y), text_one, font=font_text_one, fill=TEXT_COLOR
)
try:
- text_y = text_y + font_text_one.getsize_multiline(text_one)[1] + 16
+ text_y = text_y + text_height + 16
except (AttributeError, IndexError):
text_y = text_y + 26
if "text_two" in texts and texts["text_two"]:
- # Text one (Book subtitle)
- text_two = get_wrapped_text(texts["text_two"], font_text_two, content_width)
+ # Text two (Book subtitle)
+ text_two, text_height = get_wrapped_text(
+ texts["text_two"], font_text_two, content_width
+ )
text_layer_draw.multiline_text(
(0, text_y), text_two, font=font_text_two, fill=TEXT_COLOR
)
try:
- text_y = text_y + font_text_one.getsize_multiline(text_two)[1] + 16
+ text_y = text_y + text_height + 16
except (AttributeError, IndexError):
text_y = text_y + 26
if "text_three" in texts and texts["text_three"]:
- # Text three (Book authors)
- text_three = get_wrapped_text(
+ # Text three (Book authors, Site tagline, User address)
+ text_three, _ = get_wrapped_text(
texts["text_three"], font_text_three, content_width
)
@@ -172,7 +187,7 @@ def generate_instance_layer(content_width):
instance_text_x = 0
if logo_img:
- logo_img.thumbnail((50, 50), Image.ANTIALIAS)
+ logo_img.thumbnail((50, 50), Image.Resampling.LANCZOS)
instance_layer.paste(logo_img, (0, 0))
@@ -183,7 +198,7 @@ def generate_instance_layer(content_width):
(instance_text_x, 10), site.name, font=font_instance, fill=TEXT_COLOR
)
- line_width = 50 + 10 + font_instance.getsize(site.name)[0]
+ line_width = 50 + 10 + round(font_instance.getlength(site.name))
line_layer = Image.new(
"RGBA", (line_width, 2), color=(*(ImageColor.getrgb(TEXT_COLOR)), 50)
@@ -253,10 +268,12 @@ def generate_default_inner_img():
default_cover_draw = ImageDraw.Draw(default_cover)
text = "no image :("
- text_dimensions = font_cover.getsize(text)
+ text_left, text_top, text_right, text_bottom = font_cover.getbbox(text)
+ text_width, text_height = text_right - text_left, text_bottom - text_top
+
text_coords = (
- math.floor((inner_img_width - text_dimensions[0]) / 2),
- math.floor((inner_img_height - text_dimensions[1]) / 2),
+ math.floor((inner_img_width - text_width) / 2),
+ math.floor((inner_img_height - text_height) / 2),
)
default_cover_draw.text(text_coords, text, font=font_cover, fill="white")
@@ -273,7 +290,9 @@ def generate_preview_image(
# Cover
try:
inner_img_layer = Image.open(picture)
- inner_img_layer.thumbnail((inner_img_width, inner_img_height), Image.ANTIALIAS)
+ inner_img_layer.thumbnail(
+ (inner_img_width, inner_img_height), Image.Resampling.LANCZOS
+ )
color_thief = ColorThief(picture)
dominant_color = color_thief.get_color(quality=1)
except: # pylint: disable=bare-except
@@ -401,7 +420,7 @@ def save_and_cleanup(image, instance=None):
# pylint: disable=invalid-name
-@app.task(queue=LOW)
+@app.task(queue=IMAGES)
def generate_site_preview_image_task():
"""generate preview_image for the website"""
if not settings.ENABLE_PREVIEW_IMAGES:
@@ -426,7 +445,7 @@ def generate_site_preview_image_task():
# pylint: disable=invalid-name
-@app.task(queue=LOW)
+@app.task(queue=IMAGES)
def generate_edition_preview_image_task(book_id):
"""generate preview_image for a book"""
if not settings.ENABLE_PREVIEW_IMAGES:
@@ -451,14 +470,17 @@ def generate_edition_preview_image_task(book_id):
save_and_cleanup(image, instance=book)
-@app.task(queue=LOW)
+@app.task(queue=IMAGES)
def generate_user_preview_image_task(user_id):
- """generate preview_image for a book"""
+ """generate preview_image for a user"""
if not settings.ENABLE_PREVIEW_IMAGES:
return
user = models.User.objects.get(id=user_id)
+ if not user.local:
+ return
+
texts = {
"text_one": user.display_name,
"text_three": f"@{user.localname}@{settings.DOMAIN}",
@@ -472,3 +494,25 @@ def generate_user_preview_image_task(user_id):
image = generate_preview_image(texts=texts, picture=avatar)
save_and_cleanup(image, instance=user)
+
+
+@app.task(queue=IMAGES)
+def remove_user_preview_image_task(user_id):
+ """remove preview_image for a user"""
+ if not settings.ENABLE_PREVIEW_IMAGES:
+ return
+
+ user = models.User.objects.get(id=user_id)
+
+ try:
+ file_name = user.preview_image.name
+ except ValueError:
+ file_name = None
+
+ # Delete image in model
+ user.preview_image.delete(save=False)
+ user.save(broadcast=False, update_fields=["preview_image"])
+
+ # Delete image file
+ if file_name and default_storage.exists(file_name):
+ default_storage.delete(file_name)
diff --git a/bookwyrm/redis_store.py b/bookwyrm/redis_store.py
index ae50db2ee..e188487aa 100644
--- a/bookwyrm/redis_store.py
+++ b/bookwyrm/redis_store.py
@@ -4,12 +4,7 @@ import redis
from bookwyrm import settings
-r = redis.Redis(
- host=settings.REDIS_ACTIVITY_HOST,
- port=settings.REDIS_ACTIVITY_PORT,
- password=settings.REDIS_ACTIVITY_PASSWORD,
- db=settings.REDIS_ACTIVITY_DB_INDEX,
-)
+r = redis.from_url(settings.REDIS_ACTIVITY_URL)
class RedisStore(ABC):
@@ -21,12 +16,12 @@ class RedisStore(ABC):
"""the object and rank"""
return {obj.id: self.get_rank(obj)}
- def add_object_to_related_stores(self, obj, execute=True):
- """add an object to all suitable stores"""
+ def add_object_to_stores(self, obj, stores, execute=True):
+ """add an object to a given set of stores"""
value = self.get_value(obj)
# we want to do this as a bulk operation, hence "pipeline"
pipeline = r.pipeline()
- for store in self.get_stores_for_object(obj):
+ for store in stores:
# add the status to the feed
pipeline.zadd(store, value)
# trim the store
@@ -37,14 +32,14 @@ class RedisStore(ABC):
# and go!
return pipeline.execute()
- def remove_object_from_related_stores(self, obj, stores=None):
+ # pylint: disable=no-self-use
+ def remove_object_from_stores(self, obj, stores):
"""remove an object from all stores"""
- # if the stoers are provided, the object can just be an id
+ # if the stores are provided, the object can just be an id
if stores and isinstance(obj, int):
obj_id = obj
else:
obj_id = obj.id
- stores = self.get_stores_for_object(obj) if stores is None else stores
pipeline = r.pipeline()
for store in stores:
pipeline.zrem(store, -1, obj_id)
@@ -87,10 +82,6 @@ class RedisStore(ABC):
def get_objects_for_store(self, store):
"""a queryset of what should go in a store, used for populating it"""
- @abstractmethod
- def get_stores_for_object(self, obj):
- """the stores that an object belongs in"""
-
@abstractmethod
def get_rank(self, obj):
"""how to rank an object"""
diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py
index de898609f..94ec761db 100644
--- a/bookwyrm/settings.py
+++ b/bookwyrm/settings.py
@@ -1,9 +1,12 @@
""" bookwyrm settings and configuration """
import os
+from typing import AnyStr
+
from environs import Env
import requests
from django.utils.translation import gettext_lazy as _
+from django.core.exceptions import ImproperlyConfigured
# pylint: disable=line-too-long
@@ -11,22 +14,22 @@ from django.utils.translation import gettext_lazy as _
env = Env()
env.read_env()
DOMAIN = env("DOMAIN")
-VERSION = "0.4.6"
+VERSION = "0.6.6"
RELEASE_API = env(
"RELEASE_API",
"https://api.github.com/repos/bookwyrm-social/bookwyrm/releases/latest",
)
-PAGE_LENGTH = env("PAGE_LENGTH", 15)
+PAGE_LENGTH = env.int("PAGE_LENGTH", 15)
DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English")
-JS_CACHE = "e678183c"
+JS_CACHE = "ac315a3b"
# email
EMAIL_BACKEND = env("EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend")
EMAIL_HOST = env("EMAIL_HOST")
-EMAIL_PORT = env("EMAIL_PORT", 587)
+EMAIL_PORT = env.int("EMAIL_PORT", 587)
EMAIL_HOST_USER = env("EMAIL_HOST_USER")
EMAIL_HOST_PASSWORD = env("EMAIL_HOST_PASSWORD")
EMAIL_USE_TLS = env.bool("EMAIL_USE_TLS", True)
@@ -36,7 +39,7 @@ EMAIL_SENDER_DOMAIN = env("EMAIL_SENDER_DOMAIN", DOMAIN)
EMAIL_SENDER = f"{EMAIL_SENDER_NAME}@{EMAIL_SENDER_DOMAIN}"
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+BASE_DIR: AnyStr = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LOCALE_PATHS = [
os.path.join(BASE_DIR, "locale"),
]
@@ -68,13 +71,15 @@ FONT_DIR = os.path.join(STATIC_ROOT, "fonts")
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = env("SECRET_KEY")
-
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = env.bool("DEBUG", True)
USE_HTTPS = env.bool("USE_HTTPS", not DEBUG)
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = env("SECRET_KEY")
+if not DEBUG and SECRET_KEY == "7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr":
+ raise ImproperlyConfigured("You must change the SECRET_KEY env variable")
+
ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", ["*"])
# Application definition
@@ -101,6 +106,7 @@ MIDDLEWARE = [
"django.middleware.locale.LocaleMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
+ "csp.middleware.CSPMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"bookwyrm.middleware.TimezoneMiddleware",
"bookwyrm.middleware.IPBlocklistMiddleware",
@@ -147,6 +153,9 @@ LOGGING = {
"require_debug_true": {
"()": "django.utils.log.RequireDebugTrue",
},
+ "ignore_missing_variable": {
+ "()": "bookwyrm.utils.log.IgnoreVariableDoesNotExist",
+ },
},
"handlers": {
# Overrides the default handler to make it log to console
@@ -154,6 +163,7 @@ LOGGING = {
# console if DEBUG=False)
"console": {
"level": LOG_LEVEL,
+ "filters": ["ignore_missing_variable"],
"class": "logging.StreamHandler",
},
# This is copied as-is from the default logger, and is
@@ -189,7 +199,8 @@ STATICFILES_FINDERS = [
]
SASS_PROCESSOR_INCLUDE_FILE_PATTERN = r"^.+\.[s]{0,1}(?:a|c)ss$"
-SASS_PROCESSOR_ENABLED = True
+# when debug is disabled, make sure to compile themes once with `./bw-dev compile_themes`
+SASS_PROCESSOR_ENABLED = DEBUG
# minify css is production but not dev
if not DEBUG:
@@ -199,11 +210,14 @@ WSGI_APPLICATION = "bookwyrm.wsgi.application"
# redis/activity streams settings
REDIS_ACTIVITY_HOST = env("REDIS_ACTIVITY_HOST", "localhost")
-REDIS_ACTIVITY_PORT = env("REDIS_ACTIVITY_PORT", 6379)
-REDIS_ACTIVITY_PASSWORD = env("REDIS_ACTIVITY_PASSWORD", None)
-REDIS_ACTIVITY_DB_INDEX = env("REDIS_ACTIVITY_DB_INDEX", 0)
-
-MAX_STREAM_LENGTH = int(env("MAX_STREAM_LENGTH", 200))
+REDIS_ACTIVITY_PORT = env.int("REDIS_ACTIVITY_PORT", 6379)
+REDIS_ACTIVITY_PASSWORD = requests.utils.quote(env("REDIS_ACTIVITY_PASSWORD", ""))
+REDIS_ACTIVITY_DB_INDEX = env.int("REDIS_ACTIVITY_DB_INDEX", 0)
+REDIS_ACTIVITY_URL = env(
+ "REDIS_ACTIVITY_URL",
+ f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}",
+)
+MAX_STREAM_LENGTH = env.int("MAX_STREAM_LENGTH", 200)
STREAMS = [
{"key": "home", "name": _("Home Timeline"), "shortname": _("Home")},
@@ -212,12 +226,12 @@ STREAMS = [
# Search configuration
# total time in seconds that the instance will spend searching connectors
-SEARCH_TIMEOUT = int(env("SEARCH_TIMEOUT", 8))
+SEARCH_TIMEOUT = env.int("SEARCH_TIMEOUT", 8)
# timeout for a query to an individual connector
-QUERY_TIMEOUT = int(env("QUERY_TIMEOUT", 5))
+QUERY_TIMEOUT = env.int("INTERACTIVE_QUERY_TIMEOUT", env.int("QUERY_TIMEOUT", 5))
# Redis cache backend
-if env("USE_DUMMY_CACHE", False):
+if env.bool("USE_DUMMY_CACHE", False):
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.dummy.DummyCache",
@@ -227,7 +241,7 @@ else:
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
- "LOCATION": f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}",
+ "LOCATION": REDIS_ACTIVITY_URL,
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
},
@@ -247,7 +261,7 @@ DATABASES = {
"USER": env("POSTGRES_USER", "bookwyrm"),
"PASSWORD": env("POSTGRES_PASSWORD", "bookwyrm"),
"HOST": env("POSTGRES_HOST", ""),
- "PORT": env("PGPORT", 5432),
+ "PORT": env.int("PGPORT", 5432),
},
}
@@ -282,12 +296,15 @@ LANGUAGES = [
("en-us", _("English")),
("ca-es", _("Català (Catalan)")),
("de-de", _("Deutsch (German)")),
+ ("eo-uy", _("Esperanto (Esperanto)")),
("es-es", _("Español (Spanish)")),
+ ("eu-es", _("Euskara (Basque)")),
("gl-es", _("Galego (Galician)")),
("it-it", _("Italiano (Italian)")),
("fi-fi", _("Suomi (Finnish)")),
("fr-fr", _("Français (French)")),
("lt-lt", _("Lietuvių (Lithuanian)")),
+ ("nl-nl", _("Nederlands (Dutch)")),
("no-no", _("Norsk (Norwegian)")),
("pl-pl", _("Polski (Polish)")),
("pt-br", _("Português do Brasil (Brazilian Portuguese)")),
@@ -298,6 +315,10 @@ LANGUAGES = [
("zh-hant", _("繁體中文 (Traditional Chinese)")),
]
+LANGUAGE_ARTICLES = {
+ "English": {"the", "a", "an"},
+ "Español (Spanish)": {"un", "una", "unos", "unas", "el", "la", "los", "las"},
+}
TIME_ZONE = "UTC"
@@ -320,14 +341,18 @@ IMAGEKIT_DEFAULT_CACHEFILE_STRATEGY = "bookwyrm.thumbnail_generation.Strategy"
# https://docs.djangoproject.com/en/3.2/howto/static-files/
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+CSP_ADDITIONAL_HOSTS = env.list("CSP_ADDITIONAL_HOSTS", [])
# Storage
PROTOCOL = "http"
if USE_HTTPS:
PROTOCOL = "https"
+ SESSION_COOKIE_SECURE = True
+ CSRF_COOKIE_SECURE = True
USE_S3 = env.bool("USE_S3", False)
+USE_AZURE = env.bool("USE_AZURE", False)
if USE_S3:
# AWS settings
@@ -349,14 +374,53 @@ if USE_S3:
MEDIA_FULL_URL = MEDIA_URL
STATIC_FULL_URL = STATIC_URL
DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
+ CSP_DEFAULT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
+ CSP_SCRIPT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
+elif USE_AZURE:
+ AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME")
+ AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY")
+ AZURE_CONTAINER = env("AZURE_CONTAINER")
+ AZURE_CUSTOM_DOMAIN = env("AZURE_CUSTOM_DOMAIN")
+ # Azure Static settings
+ STATIC_LOCATION = "static"
+ STATIC_URL = (
+ f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{STATIC_LOCATION}/"
+ )
+ STATICFILES_STORAGE = "bookwyrm.storage_backends.AzureStaticStorage"
+ # Azure Media settings
+ MEDIA_LOCATION = "images"
+ MEDIA_URL = (
+ f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{MEDIA_LOCATION}/"
+ )
+ MEDIA_FULL_URL = MEDIA_URL
+ STATIC_FULL_URL = STATIC_URL
+ DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.AzureImagesStorage"
+ CSP_DEFAULT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
+ CSP_SCRIPT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
else:
STATIC_URL = "/static/"
MEDIA_URL = "/images/"
MEDIA_FULL_URL = f"{PROTOCOL}://{DOMAIN}{MEDIA_URL}"
STATIC_FULL_URL = f"{PROTOCOL}://{DOMAIN}{STATIC_URL}"
+ CSP_DEFAULT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
+ CSP_SCRIPT_SRC = ["'self'"] + CSP_ADDITIONAL_HOSTS
+
+CSP_INCLUDE_NONCE_IN = ["script-src"]
OTEL_EXPORTER_OTLP_ENDPOINT = env("OTEL_EXPORTER_OTLP_ENDPOINT", None)
OTEL_EXPORTER_OTLP_HEADERS = env("OTEL_EXPORTER_OTLP_HEADERS", None)
OTEL_SERVICE_NAME = env("OTEL_SERVICE_NAME", None)
+OTEL_EXPORTER_CONSOLE = env.bool("OTEL_EXPORTER_CONSOLE", False)
-TWO_FACTOR_LOGIN_MAX_SECONDS = 60
+TWO_FACTOR_LOGIN_MAX_SECONDS = env.int("TWO_FACTOR_LOGIN_MAX_SECONDS", 60)
+TWO_FACTOR_LOGIN_VALIDITY_WINDOW = env.int("TWO_FACTOR_LOGIN_VALIDITY_WINDOW", 2)
+
+HTTP_X_FORWARDED_PROTO = env.bool("SECURE_PROXY_SSL_HEADER", False)
+if HTTP_X_FORWARDED_PROTO:
+ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+# Instance Actor for signing GET requests to "secure mode"
+# Mastodon servers.
+# Do not change this setting unless you already have an existing
+# user with the same username - in which case you should change it!
+INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"
diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py
index 61cafe71f..08780b731 100644
--- a/bookwyrm/signatures.py
+++ b/bookwyrm/signatures.py
@@ -15,29 +15,40 @@ MAX_SIGNATURE_AGE = 300
def create_key_pair():
"""a new public/private key pair, used for creating new users"""
random_generator = Random.new().read
- key = RSA.generate(1024, random_generator)
+ key = RSA.generate(2048, random_generator)
private_key = key.export_key().decode("utf8")
- public_key = key.publickey().export_key().decode("utf8")
+ public_key = key.public_key().export_key().decode("utf8")
return private_key, public_key
-def make_signature(sender, destination, date, digest):
+def make_signature(method, sender, destination, date, **kwargs):
"""uses a private key to sign an outgoing message"""
inbox_parts = urlparse(destination)
signature_headers = [
- f"(request-target): post {inbox_parts.path}",
+ f"(request-target): {method} {inbox_parts.path}",
f"host: {inbox_parts.netloc}",
f"date: {date}",
- f"digest: {digest}",
]
+ headers = "(request-target) host date"
+ digest = kwargs.get("digest")
+ if digest is not None:
+ signature_headers.append(f"digest: {digest}")
+ headers = "(request-target) host date digest"
+
message_to_sign = "\n".join(signature_headers)
signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key))
signed_message = signer.sign(SHA256.new(message_to_sign.encode("utf8")))
+ # For legacy reasons we need to use an incorrect keyId for older Bookwyrm versions
+ key_id = (
+ f"{sender.remote_id}#main-key"
+ if kwargs.get("use_legacy_key")
+ else f"{sender.remote_id}/#main-key"
+ )
signature = {
- "keyId": f"{sender.remote_id}#main-key",
+ "keyId": key_id,
"algorithm": "rsa-sha256",
- "headers": "(request-target) host date digest",
+ "headers": headers,
"signature": b64encode(signed_message).decode("utf8"),
}
return ",".join(f'{k}="{v}"' for (k, v) in signature.items())
diff --git a/bookwyrm/static/css/bookwyrm/_all.scss b/bookwyrm/static/css/bookwyrm/_all.scss
index f1b23db9a..5bb08b931 100644
--- a/bookwyrm/static/css/bookwyrm/_all.scss
+++ b/bookwyrm/static/css/bookwyrm/_all.scss
@@ -137,6 +137,10 @@ button:focus-visible .button-invisible-overlay {
opacity: 1;
}
+button.button-paragraph {
+ vertical-align: middle;
+}
+
/** States
******************************************************************************/
diff --git a/bookwyrm/static/css/bookwyrm/components/_book_cover.scss b/bookwyrm/static/css/bookwyrm/components/_book_cover.scss
index d1125197e..48b564a0b 100644
--- a/bookwyrm/static/css/bookwyrm/components/_book_cover.scss
+++ b/bookwyrm/static/css/bookwyrm/components/_book_cover.scss
@@ -5,7 +5,7 @@
* - .book-cover is positioned and sized based on its container.
*
* To have the cover within specific dimensions, specify a width or height for
- * standard bulma’s named breapoints:
+ * standard bulma’s named breakpoints:
*
* `is-(w|h)-(auto|xs|s|m|l|xl|xxl)[-(mobile|tablet|desktop)]`
*
@@ -43,7 +43,7 @@
max-height: 100%;
/* Useful when stretching under-sized images. */
- image-rendering: optimizequality;
+ image-rendering: optimizeQuality;
image-rendering: smooth;
}
diff --git a/bookwyrm/static/css/bookwyrm/components/_copy.scss b/bookwyrm/static/css/bookwyrm/components/_copy.scss
index e0c4246e6..7a47c1dba 100644
--- a/bookwyrm/static/css/bookwyrm/components/_copy.scss
+++ b/bookwyrm/static/css/bookwyrm/components/_copy.scss
@@ -28,3 +28,31 @@
.vertical-copy button {
width: 100%;
}
+
+.copy-tooltip {
+ overflow: visible;
+ visibility: hidden;
+ width: 140px;
+ background-color: #555;
+ color: #fff;
+ text-align: center;
+ border-radius: 6px;
+ padding: 5px;
+ position: absolute;
+ z-index: 1;
+ margin-left: -30px;
+ margin-top: -45px;
+ opacity: 0;
+ transition: opacity 0.3s;
+}
+
+.copy-tooltip::after {
+ content: "";
+ position: absolute;
+ top: 100%;
+ left: 50%;
+ margin-left: -60px;
+ border-width: 5px;
+ border-style: solid;
+ border-color: #555 transparent transparent transparent;
+ }
diff --git a/bookwyrm/static/css/bookwyrm/components/_details.scss b/bookwyrm/static/css/bookwyrm/components/_details.scss
index de29629c8..4145554eb 100644
--- a/bookwyrm/static/css/bookwyrm/components/_details.scss
+++ b/bookwyrm/static/css/bookwyrm/components/_details.scss
@@ -81,7 +81,19 @@ details.dropdown .dropdown-menu a:focus-visible {
details.details-panel {
box-shadow: 0 0 0 1px $border;
transition: box-shadow 0.2s ease;
- padding: 0.75rem;
+ padding: 0;
+
+ > * {
+ padding: 0.75rem;
+ }
+
+ summary {
+ position: relative;
+
+ .details-close {
+ padding: 0.75rem;
+ }
+ }
}
details[open].details-panel,
@@ -89,10 +101,6 @@ details.details-panel:hover {
box-shadow: 0 0 0 1px $border;
}
-details.details-panel summary {
- position: relative;
-}
-
details summary .details-close {
position: absolute;
right: 0;
diff --git a/bookwyrm/static/css/bookwyrm/components/_stars.scss b/bookwyrm/static/css/bookwyrm/components/_stars.scss
index 1a8e3680f..db2772dc0 100644
--- a/bookwyrm/static/css/bookwyrm/components/_stars.scss
+++ b/bookwyrm/static/css/bookwyrm/components/_stars.scss
@@ -5,6 +5,10 @@
white-space: nowrap;
}
+.stars .no-rating {
+ font-style: italic;
+}
+
/** Stars in a review form
*
* Specificity makes hovering taking over checked inputs.
diff --git a/bookwyrm/static/css/bookwyrm/components/_tabs.scss b/bookwyrm/static/css/bookwyrm/components/_tabs.scss
index 8e00f6a88..2d68a383b 100644
--- a/bookwyrm/static/css/bookwyrm/components/_tabs.scss
+++ b/bookwyrm/static/css/bookwyrm/components/_tabs.scss
@@ -44,12 +44,12 @@
.bw-tabs a:hover {
border-bottom-color: transparent;
- color: $text;
+ color: $text
}
.bw-tabs a.is-active {
border-bottom-color: transparent;
- color: $link;
+ color: $link
}
.bw-tabs.is-left {
diff --git a/bookwyrm/static/css/bookwyrm/overrides/_bulma_overrides.scss b/bookwyrm/static/css/bookwyrm/overrides/_bulma_overrides.scss
index f46e7b957..9ab44f89d 100644
--- a/bookwyrm/static/css/bookwyrm/overrides/_bulma_overrides.scss
+++ b/bookwyrm/static/css/bookwyrm/overrides/_bulma_overrides.scss
@@ -1,3 +1,53 @@
+.summary-on-open {
+ display: none;
+}
+
+@media only screen and (max-width: 768px) {
+ .navbar-menu {
+ text-align: right;
+ padding-right: 1rem;
+
+ .tags {
+ justify-content: flex-end;
+ }
+
+ #navbar-dropdown {
+ &[open] {
+ .summary-on-open {
+ display: initial;
+ position: fixed;
+ top: 0;
+ left: 0;
+ right: 0;
+ height: 3rem;
+ z-index: 31;
+ background-color: $dropdown-content-background-color;
+ padding: 1rem 1.75rem;
+ line-height: 1;
+ }
+ }
+
+ .dropdown-menu {
+ padding-top: 0;
+ top: 3rem;
+ }
+
+ .dropdown-content {
+ padding-top: 0;
+ box-shadow: none;
+ border-top-left-radius: 0;
+ border-top-right-radius: 0;
+ }
+
+ .navbar-item {
+ /* see ../components/_details.scss :: Navbar details */
+ padding-right: 1.75rem;
+ font-size: 1rem;
+ }
+ }
+ }
+}
+
.image {
overflow: hidden;
}
@@ -59,3 +109,9 @@
max-height: 35em;
overflow: hidden;
}
+
+.dropdown-menu .button {
+ @include mobile {
+ font-size: $size-6;
+ }
+}
diff --git a/bookwyrm/static/css/bookwyrm/utilities/_size.scss b/bookwyrm/static/css/bookwyrm/utilities/_size.scss
index cbc74d7ab..258aa9a73 100644
--- a/bookwyrm/static/css/bookwyrm/utilities/_size.scss
+++ b/bookwyrm/static/css/bookwyrm/utilities/_size.scss
@@ -40,6 +40,10 @@
width: 500px !important;
}
+.is-h-em {
+ height: 1em !important;
+}
+
.is-h-xs {
height: 80px !important;
}
diff --git a/bookwyrm/static/css/fonts/icomoon.eot b/bookwyrm/static/css/fonts/icomoon.eot
index 69628662b..33dc07eec 100644
Binary files a/bookwyrm/static/css/fonts/icomoon.eot and b/bookwyrm/static/css/fonts/icomoon.eot differ
diff --git a/bookwyrm/static/css/fonts/icomoon.svg b/bookwyrm/static/css/fonts/icomoon.svg
index c67c8b225..058c19226 100644
--- a/bookwyrm/static/css/fonts/icomoon.svg
+++ b/bookwyrm/static/css/fonts/icomoon.svg
@@ -39,9 +39,12 @@
+
+
+
diff --git a/bookwyrm/static/css/fonts/icomoon.ttf b/bookwyrm/static/css/fonts/icomoon.ttf
index 12c79d551..89d3be8fa 100644
Binary files a/bookwyrm/static/css/fonts/icomoon.ttf and b/bookwyrm/static/css/fonts/icomoon.ttf differ
diff --git a/bookwyrm/static/css/fonts/icomoon.woff b/bookwyrm/static/css/fonts/icomoon.woff
index 624b70f33..95325ab4a 100644
Binary files a/bookwyrm/static/css/fonts/icomoon.woff and b/bookwyrm/static/css/fonts/icomoon.woff differ
diff --git a/bookwyrm/static/css/themes/bookwyrm-dark.scss b/bookwyrm/static/css/themes/bookwyrm-dark.scss
index 928eadd48..df756fd02 100644
--- a/bookwyrm/static/css/themes/bookwyrm-dark.scss
+++ b/bookwyrm/static/css/themes/bookwyrm-dark.scss
@@ -15,6 +15,8 @@ $danger: #872538;
$danger-light: #481922;
$light: #393939;
$red: #ffa1b4;
+$black: #000;
+$white-ter: hsl(0, 0%, 90%);
/* book cover standins */
$no-cover-color: #002549;
@@ -56,9 +58,12 @@ $link-active: $white-bis;
$link-light: #0d1c26;
/* bulma overrides */
+$body-background-color: rgb(17, 18, 18);
$background: $background-secondary;
$menu-item-active-background-color: $link-background;
$navbar-dropdown-item-hover-color: $white;
+$info-light: $background-body;
+$info-dark: #72b6ee;
/* These element's colors are hardcoded, probably a bug in bulma? */
@media screen and (min-width: 769px) {
@@ -74,7 +79,7 @@ $navbar-dropdown-item-hover-color: $white;
}
/* misc */
-$shadow: 0 0.5em 1em -0.125em rgba($black, 0.2), 0 0 0 1px rgba($black, 0.02);
+$shadow: 0 0.5em 0.5em -0.125em rgba($black, 0.2), 0 0 0 1px rgba($black, 0.02);
$card-header-shadow: 0 0.125em 0.25em rgba($black, 0.1);
$invisible-overlay-background-color: rgba($black, 0.66);
$progress-value-background-color: $border-light;
@@ -92,6 +97,26 @@ $family-secondary: $family-sans-serif;
color: $grey-light !important;
}
+.tabs li:not(.is-active) a {
+ color: #2e7eb9 !important;
+}
+ .tabs li:not(.is-active) a:hover {
+ border-bottom-color: #2e7eb9 !important;
+}
+
+.tabs li:not(.is-active) a {
+ color: #2e7eb9 !important;
+}
+.tabs li.is-active a {
+ color: #e6e6e6 !important;
+ border-bottom-color: #e6e6e6 !important ;
+}
+
+
+#qrcode svg {
+ background-color: #a6a6a6;
+}
+
@import "../bookwyrm";
@import "../vendor/icons.css";
-@import "../vendor/shepherd.scss";
+@import "../vendor/shepherd";
diff --git a/bookwyrm/static/css/themes/bookwyrm-light.scss b/bookwyrm/static/css/themes/bookwyrm-light.scss
index 5b0630379..4a3f16a27 100644
--- a/bookwyrm/static/css/themes/bookwyrm-light.scss
+++ b/bookwyrm/static/css/themes/bookwyrm-light.scss
@@ -66,6 +66,22 @@ $family-secondary: $family-sans-serif;
color: $grey !important;
}
+.tabs li:not(.is-active) a {
+ color: #3273dc !important;
+}
+ .tabs li:not(.is-active) a:hover {
+ border-bottom-color: #3273dc !important;
+}
+
+.tabs li:not(.is-active) a {
+ color: #3273dc !important;
+}
+.tabs li.is-active a {
+ color: #4a4a4a !important;
+ border-bottom-color: #4a4a4a !important ;
+}
+
+
@import "../bookwyrm";
@import "../vendor/icons.css";
-@import "../vendor/shepherd.scss";
+@import "../vendor/shepherd";
diff --git a/bookwyrm/static/css/vendor/icons.css b/bookwyrm/static/css/vendor/icons.css
index 6477aee5c..6af5c2813 100644
--- a/bookwyrm/static/css/vendor/icons.css
+++ b/bookwyrm/static/css/vendor/icons.css
@@ -1,10 +1,10 @@
@font-face {
font-family: 'icomoon';
- src: url('../fonts/icomoon.eot?r7jc98');
- src: url('../fonts/icomoon.eot?r7jc98#iefix') format('embedded-opentype'),
- url('../fonts/icomoon.ttf?r7jc98') format('truetype'),
- url('../fonts/icomoon.woff?r7jc98') format('woff'),
- url('../fonts/icomoon.svg?r7jc98#icomoon') format('svg');
+ src: url('../fonts/icomoon.eot?nr4nq7');
+ src: url('../fonts/icomoon.eot?nr4nq7#iefix') format('embedded-opentype'),
+ url('../fonts/icomoon.ttf?nr4nq7') format('truetype'),
+ url('../fonts/icomoon.woff?nr4nq7') format('woff'),
+ url('../fonts/icomoon.svg?nr4nq7#icomoon') format('svg');
font-weight: normal;
font-style: normal;
font-display: block;
@@ -122,6 +122,9 @@
.icon-graphic-banknote:before {
content: "\e920";
}
+.icon-copy:before {
+ content: "\e92c";
+}
.icon-search:before {
content: "\e986";
}
@@ -152,3 +155,9 @@
.icon-barcode:before {
content: "\e937";
}
+.icon-eye:before {
+ content: "\e9ce";
+}
+.icon-eye-blocked:before {
+ content: "\e9d1";
+}
diff --git a/bookwyrm/static/css/vendor/shepherd.scss b/bookwyrm/static/css/vendor/shepherd.scss
index f8d39b782..5e84b2ea7 100644
--- a/bookwyrm/static/css/vendor/shepherd.scss
+++ b/bookwyrm/static/css/vendor/shepherd.scss
@@ -6,16 +6,16 @@
@use 'bulma/bulma.sass';
.shepherd-button {
- @extend .button.mr-2;
+ @extend .button, .mr-2;
}
.shepherd-button.shepherd-button-secondary {
- @extend .button.is-light;
+ @extend .button, .is-light;
}
.shepherd-footer {
@extend .message-body;
- @extend .is-info.is-light;
+ @extend .is-info, .is-light;
border-color: $info-light;
border-radius: 0 0 4px 4px;
}
@@ -29,7 +29,7 @@
.shepherd-text {
@extend .message-body;
- @extend .is-info.is-light;
+ @extend .is-info, .is-light;
border-radius: 0;
}
diff --git a/bookwyrm/static/js/autocomplete.js b/bookwyrm/static/js/autocomplete.js
index 84474e43c..a98cd9634 100644
--- a/bookwyrm/static/js/autocomplete.js
+++ b/bookwyrm/static/js/autocomplete.js
@@ -106,7 +106,7 @@ const tries = {
e: {
p: {
u: {
- b: "ePub",
+ b: "EPUB",
},
},
},
diff --git a/bookwyrm/static/js/bookwyrm.js b/bookwyrm/static/js/bookwyrm.js
index aa06a8b0a..a2351a98c 100644
--- a/bookwyrm/static/js/bookwyrm.js
+++ b/bookwyrm/static/js/bookwyrm.js
@@ -5,7 +5,7 @@ let BookWyrm = new (class {
constructor() {
this.MAX_FILE_SIZE_BYTES = 10 * 1000000;
this.initOnDOMLoaded();
- this.initReccuringTasks();
+ this.initRecurringTasks();
this.initEventListeners();
}
@@ -30,6 +30,12 @@ let BookWyrm = new (class {
.querySelectorAll("[data-back]")
.forEach((button) => button.addEventListener("click", this.back));
+ document
+ .querySelectorAll("[data-password-icon]")
+ .forEach((button) =>
+ button.addEventListener("click", this.togglePasswordVisibility.bind(this))
+ );
+
document
.querySelectorAll('input[type="file"]')
.forEach((node) => node.addEventListener("change", this.disableIfTooLarge.bind(this)));
@@ -40,14 +46,17 @@ let BookWyrm = new (class {
document.querySelectorAll("details.dropdown").forEach((node) => {
node.addEventListener("toggle", this.handleDetailsDropdown.bind(this));
- node.querySelectorAll("[data-modal-open]").forEach((modal_node) =>
- modal_node.addEventListener("click", () => (node.open = false))
- );
});
document
.querySelector("#barcode-scanner-modal")
.addEventListener("open", this.openBarcodeScanner.bind(this));
+
+ document
+ .querySelectorAll('form[name="register"]')
+ .forEach((form) =>
+ form.addEventListener("submit", (e) => this.setPreferredTimezone(e, form))
+ );
}
/**
@@ -62,6 +71,9 @@ let BookWyrm = new (class {
.querySelectorAll('input[type="file"]')
.forEach(bookwyrm.disableIfTooLarge.bind(bookwyrm));
document.querySelectorAll("[data-copytext]").forEach(bookwyrm.copyText.bind(bookwyrm));
+ document
+ .querySelectorAll("[data-copywithtooltip]")
+ .forEach(bookwyrm.copyWithTooltip.bind(bookwyrm));
document
.querySelectorAll(".modal.is-active")
.forEach(bookwyrm.handleActiveModal.bind(bookwyrm));
@@ -71,7 +83,7 @@ let BookWyrm = new (class {
/**
* Execute recurring tasks.
*/
- initReccuringTasks() {
+ initRecurringTasks() {
// Polling
document.querySelectorAll("[data-poll]").forEach((liveArea) => this.polling(liveArea));
}
@@ -89,7 +101,6 @@ let BookWyrm = new (class {
/**
* Update a counter with recurring requests to the API
- * The delay is slightly randomized and increased on each cycle.
*
* @param {Object} counter - DOM node
* @param {int} delay - frequency for polling in ms
@@ -98,16 +109,19 @@ let BookWyrm = new (class {
polling(counter, delay) {
const bookwyrm = this;
- delay = delay || 10000;
- delay += Math.random() * 1000;
+ delay = delay || 5 * 60 * 1000 + (Math.random() - 0.5) * 30 * 1000;
setTimeout(
function () {
fetch("/api/updates/" + counter.dataset.poll)
.then((response) => response.json())
- .then((data) => bookwyrm.updateCountElement(counter, data));
-
- bookwyrm.polling(counter, delay * 1.25);
+ .then((data) => {
+ bookwyrm.updateCountElement(counter, data);
+ bookwyrm.polling(counter);
+ })
+ .catch(() => {
+ bookwyrm.polling(counter, delay * 1.1);
+ });
},
delay,
counter
@@ -519,6 +533,21 @@ let BookWyrm = new (class {
textareaEl.parentNode.appendChild(copyButtonEl);
}
+ copyWithTooltip(copyButtonEl) {
+ const text = document.getElementById(copyButtonEl.dataset.contentId).innerHTML;
+ const tooltipEl = document.getElementById(copyButtonEl.dataset.tooltipId);
+
+ copyButtonEl.addEventListener("click", () => {
+ navigator.clipboard.writeText(text);
+ tooltipEl.style.visibility = "visible";
+ tooltipEl.style.opacity = 1;
+ setTimeout(function () {
+ tooltipEl.style.visibility = "hidden";
+ tooltipEl.style.opacity = 0;
+ }, 3000);
+ });
+ }
+
/**
* Handle the details dropdown component.
*
@@ -628,9 +657,9 @@ let BookWyrm = new (class {
}
function toggleStatus(status) {
- for (const child of statusNode.children) {
- BookWyrm.toggleContainer(child, !child.classList.contains(status));
- }
+ const template = document.querySelector(`#barcode-${status}`);
+
+ statusNode.replaceChildren(template ? template.content.cloneNode(true) : null);
}
function initBarcodes(cameraId = null) {
@@ -785,4 +814,36 @@ let BookWyrm = new (class {
initBarcodes();
}
+
+ /**
+ * Set preferred timezone in register form.
+ *
+ * @param {Event} event - `submit` event fired by the register form.
+ * @return {undefined}
+ */
+ setPreferredTimezone(event, form) {
+ const tz = Intl.DateTimeFormat().resolvedOptions().timeZone;
+
+ form.querySelector('input[name="preferred_timezone"]').value = tz;
+ }
+
+ togglePasswordVisibility(event) {
+ const iconElement = event.currentTarget.getElementsByTagName("button")[0];
+ const passwordElementId = event.currentTarget.dataset.for;
+ const passwordInputElement = document.getElementById(passwordElementId);
+
+ if (!passwordInputElement) return;
+
+ if (passwordInputElement.type === "password") {
+ passwordInputElement.type = "text";
+ this.addRemoveClass(iconElement, "icon-eye-blocked");
+ this.addRemoveClass(iconElement, "icon-eye", true);
+ } else {
+ passwordInputElement.type = "password";
+ this.addRemoveClass(iconElement, "icon-eye");
+ this.addRemoveClass(iconElement, "icon-eye-blocked", true);
+ }
+
+ this.toggleFocus(passwordElementId);
+ }
})();
diff --git a/bookwyrm/static/js/forms.js b/bookwyrm/static/js/forms.js
index 998873898..4a075506e 100644
--- a/bookwyrm/static/js/forms.js
+++ b/bookwyrm/static/js/forms.js
@@ -2,7 +2,7 @@
"use strict";
/**
- * Remoev input field
+ * Remove input field
*
* @param {event} the button click event
*/
@@ -46,4 +46,15 @@
document
.querySelectorAll("[data-remove]")
.forEach((node) => node.addEventListener("click", removeInput));
+
+ // Get element, add a keypress listener...
+ document.getElementById("subjects").addEventListener("keypress", function (e) {
+ // Linstening to element e.target
+ // If e.target is an input field within "subjects" div preventDefault()
+ if (e.target && e.target.nodeName == "INPUT") {
+ if (event.keyCode == 13) {
+ event.preventDefault();
+ }
+ }
+ });
})();
diff --git a/bookwyrm/storage_backends.py b/bookwyrm/storage_backends.py
index 4fb0feff0..6dd9f522c 100644
--- a/bookwyrm/storage_backends.py
+++ b/bookwyrm/storage_backends.py
@@ -2,6 +2,7 @@
import os
from tempfile import SpooledTemporaryFile
from storages.backends.s3boto3 import S3Boto3Storage
+from storages.backends.azure_storage import AzureStorage
class StaticStorage(S3Boto3Storage): # pylint: disable=abstract-method
@@ -47,3 +48,16 @@ class ImagesStorage(S3Boto3Storage): # pylint: disable=abstract-method
# Upload the object which will auto close the
# content_autoclose instance
return super()._save(name, content_autoclose)
+
+
+class AzureStaticStorage(AzureStorage): # pylint: disable=abstract-method
+ """Storage class for Static contents"""
+
+ location = "static"
+
+
+class AzureImagesStorage(AzureStorage): # pylint: disable=abstract-method
+ """Storage class for Image files"""
+
+ location = "images"
+ overwrite_files = False
diff --git a/bookwyrm/suggested_users.py b/bookwyrm/suggested_users.py
index 91f23dded..3e9bef9c4 100644
--- a/bookwyrm/suggested_users.py
+++ b/bookwyrm/suggested_users.py
@@ -4,13 +4,16 @@ import logging
from django.dispatch import receiver
from django.db import transaction
from django.db.models import signals, Count, Q, Case, When, IntegerField
+from opentelemetry import trace
from bookwyrm import models
from bookwyrm.redis_store import RedisStore, r
-from bookwyrm.tasks import app, LOW, MEDIUM
+from bookwyrm.tasks import app, SUGGESTED_USERS
+from bookwyrm.telemetry import open_telemetry
logger = logging.getLogger(__name__)
+tracer = open_telemetry.tracer()
class SuggestedUsers(RedisStore):
@@ -49,30 +52,34 @@ class SuggestedUsers(RedisStore):
)
def get_stores_for_object(self, obj):
+ """the stores that an object belongs in"""
return [self.store_id(u) for u in self.get_users_for_object(obj)]
def get_users_for_object(self, obj): # pylint: disable=no-self-use
"""given a user, who might want to follow them"""
- return models.User.objects.filter(local=True,).exclude(
+ return models.User.objects.filter(local=True, is_active=True).exclude(
Q(id=obj.id) | Q(followers=obj) | Q(id__in=obj.blocks.all()) | Q(blocks=obj)
)
+ @tracer.start_as_current_span("SuggestedUsers.rerank_obj")
def rerank_obj(self, obj, update_only=True):
"""update all the instances of this user with new ranks"""
+ trace.get_current_span().set_attribute("update_only", update_only)
pipeline = r.pipeline()
for store_user in self.get_users_for_object(obj):
- annotated_user = get_annotated_users(
- store_user,
- id=obj.id,
- ).first()
- if not annotated_user:
- continue
+ with tracer.start_as_current_span("SuggestedUsers.rerank_obj/user") as _:
+ annotated_user = get_annotated_users(
+ store_user,
+ id=obj.id,
+ ).first()
+ if not annotated_user:
+ continue
- pipeline.zadd(
- self.store_id(store_user),
- self.get_value(annotated_user),
- xx=update_only,
- )
+ pipeline.zadd(
+ self.store_id(store_user),
+ self.get_value(annotated_user),
+ xx=update_only,
+ )
pipeline.execute()
def rerank_user_suggestions(self, user):
@@ -237,41 +244,46 @@ def domain_level_update(sender, instance, created, update_fields=None, **kwargs)
# ------------------- TASKS
-@app.task(queue=LOW)
+@app.task(queue=SUGGESTED_USERS)
def rerank_suggestions_task(user_id):
"""do the hard work in celery"""
suggested_users.rerank_user_suggestions(user_id)
-@app.task(queue=LOW)
+@app.task(queue=SUGGESTED_USERS)
def rerank_user_task(user_id, update_only=False):
"""do the hard work in celery"""
user = models.User.objects.get(id=user_id)
- suggested_users.rerank_obj(user, update_only=update_only)
+ if user:
+ suggested_users.rerank_obj(user, update_only=update_only)
-@app.task(queue=LOW)
+@app.task(queue=SUGGESTED_USERS)
def remove_user_task(user_id):
"""do the hard work in celery"""
user = models.User.objects.get(id=user_id)
- suggested_users.remove_object_from_related_stores(user)
+ suggested_users.remove_object_from_stores(
+ user, suggested_users.get_stores_for_object(user)
+ )
-@app.task(queue=MEDIUM)
+@app.task(queue=SUGGESTED_USERS)
def remove_suggestion_task(user_id, suggested_user_id):
"""remove a specific user from a specific user's suggestions"""
suggested_user = models.User.objects.get(id=suggested_user_id)
suggested_users.remove_suggestion(user_id, suggested_user)
-@app.task(queue=LOW)
+@app.task(queue=SUGGESTED_USERS)
def bulk_remove_instance_task(instance_id):
"""remove a bunch of users from recs"""
for user in models.User.objects.filter(federated_server__id=instance_id):
- suggested_users.remove_object_from_related_stores(user)
+ suggested_users.remove_object_from_stores(
+ user, suggested_users.get_stores_for_object(user)
+ )
-@app.task(queue=LOW)
+@app.task(queue=SUGGESTED_USERS)
def bulk_add_instance_task(instance_id):
"""remove a bunch of users from recs"""
for user in models.User.objects.filter(federated_server__id=instance_id):
diff --git a/bookwyrm/tasks.py b/bookwyrm/tasks.py
index 09e1d267e..79e1b6340 100644
--- a/bookwyrm/tasks.py
+++ b/bookwyrm/tasks.py
@@ -10,7 +10,19 @@ app = Celery(
"tasks", broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND
)
-# priorities
+# priorities - for backwards compatibility, will be removed next release
LOW = "low_priority"
MEDIUM = "medium_priority"
HIGH = "high_priority"
+
+STREAMS = "streams"
+IMAGES = "images"
+SUGGESTED_USERS = "suggested_users"
+EMAIL = "email"
+CONNECTORS = "connectors"
+LISTS = "lists"
+INBOX = "inbox"
+IMPORTS = "imports"
+IMPORT_TRIGGERED = "import_triggered"
+BROADCAST = "broadcast"
+MISC = "misc"
diff --git a/bookwyrm/telemetry/open_telemetry.py b/bookwyrm/telemetry/open_telemetry.py
index 0b38a04b1..2a0168ff3 100644
--- a/bookwyrm/telemetry/open_telemetry.py
+++ b/bookwyrm/telemetry/open_telemetry.py
@@ -1,22 +1,41 @@
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace import TracerProvider, Tracer
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+from bookwyrm import settings
trace.set_tracer_provider(TracerProvider())
-trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+if settings.OTEL_EXPORTER_CONSOLE:
+ trace.get_tracer_provider().add_span_processor(
+ BatchSpanProcessor(ConsoleSpanExporter())
+ )
+elif settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+ trace.get_tracer_provider().add_span_processor(
+ BatchSpanProcessor(OTLPSpanExporter())
+ )
-def instrumentDjango():
+def instrumentDjango() -> None:
from opentelemetry.instrumentation.django import DjangoInstrumentor
DjangoInstrumentor().instrument()
-def instrumentCelery():
+def instrumentPostgres() -> None:
+ from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
+
+ Psycopg2Instrumentor().instrument()
+
+
+def instrumentCelery() -> None:
from opentelemetry.instrumentation.celery import CeleryInstrumentor
from celery.signals import worker_process_init
@worker_process_init.connect(weak=False)
def init_celery_tracing(*args, **kwargs):
CeleryInstrumentor().instrument()
+
+
+def tracer() -> Tracer:
+ return trace.get_tracer(__name__)
diff --git a/bookwyrm/templates/about/about.html b/bookwyrm/templates/about/about.html
index 481ecda99..6705793d5 100644
--- a/bookwyrm/templates/about/about.html
+++ b/bookwyrm/templates/about/about.html
@@ -10,8 +10,9 @@
{% endblock %}
{% block about_content %}
+{% get_current_language as LANGUAGE_CODE %}
{# seven day cache #}
-{% cache 604800 about_page %}
+{% cache 604800 about_page_superlatives LANGUAGE_CODE %}
{% get_book_superlatives as superlatives %}
+{% endcache %}
-{% endcache %}
{% endblock %}
diff --git a/bookwyrm/templates/about/impressum.html b/bookwyrm/templates/about/impressum.html
new file mode 100644
index 000000000..3f892c7a7
--- /dev/null
+++ b/bookwyrm/templates/about/impressum.html
@@ -0,0 +1,15 @@
+{% extends 'about/layout.html' %}
+{% load i18n %}
+
+{% block title %}{% trans "Impressum" %}{% endblock %}
+
+
+{% block about_content %}
+
+
{% trans "Impressum" %}
+
+ {{ site.impressum | safe }}
+
+
+
+{% endblock %}
diff --git a/bookwyrm/templates/about/layout.html b/bookwyrm/templates/about/layout.html
index e921fcd29..22237508c 100644
--- a/bookwyrm/templates/about/layout.html
+++ b/bookwyrm/templates/about/layout.html
@@ -47,6 +47,14 @@
{% trans "Privacy Policy" %}
+ {% if site.show_impressum %}
+
+ {% url 'impressum' as path %}
+
+ {% trans "Impressum" %}
+
+
+ {% endif %}
diff --git a/bookwyrm/templates/annual_summary/layout.html b/bookwyrm/templates/annual_summary/layout.html
index 3d1796250..8d399c212 100644
--- a/bookwyrm/templates/annual_summary/layout.html
+++ b/bookwyrm/templates/annual_summary/layout.html
@@ -53,7 +53,7 @@
{% trans "Share this page" %}
-