Merge pull request #3369 from bookwyrm-social/dependabot/pip/requests-2.32.0

Bump requests from 2.31.0 to 2.32.0

.dockerignore

@@ -5,3 +5,4 @@ __pycache__
 .git
 .github
 .pytest*
+.env

.env.example

@@ -8,7 +8,7 @@ USE_HTTPS=true
 DOMAIN=your.domain.here
 EMAIL=your@email.here
 
-# Instance defualt language (see options at bookwyrm/settings.py "LANGUAGES"
+# Instance default language (see options at bookwyrm/settings.py "LANGUAGES"
 LANGUAGE_CODE="en-us"
 # Used for deciding which editions to prefer
 DEFAULT_LANGUAGE="English"
@@ -16,13 +16,18 @@ DEFAULT_LANGUAGE="English"
 ## Leave unset to allow all hosts
 # ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
 
+# Specify when the site is served from a port that is not the default
+# for the protocol (80 for HTTP or 443 for HTTPS).
+# Probably only necessary in development.
+# PORT=1333
+
 MEDIA_ROOT=images/
 
 # Database configuration
 PGPORT=5432
 POSTGRES_PASSWORD=securedbypassword123
-POSTGRES_USER=fedireads
-POSTGRES_DB=fedireads
+POSTGRES_USER=bookwyrm
+POSTGRES_DB=bookwyrm
 POSTGRES_HOST=db
 
 # Redis activity stream manager
@@ -32,12 +37,17 @@ REDIS_ACTIVITY_PORT=6379
 REDIS_ACTIVITY_PASSWORD=redispassword345
 # Optional, use a different redis database (defaults to 0)
 # REDIS_ACTIVITY_DB_INDEX=0
+# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
+# REDIS_ACTIVITY_URL=
 
 # Redis as celery broker
+REDIS_BROKER_HOST=redis_broker
 REDIS_BROKER_PORT=6379
 REDIS_BROKER_PASSWORD=redispassword123
 # Optional, use a different redis database (defaults to 0)
 # REDIS_BROKER_DB_INDEX=0
+# Alternatively specify the full redis url, i.e. if you need to use a unix:// socket
+# REDIS_BROKER_URL=
 
 # Monitoring for celery
 FLOWER_PORT=8888
@@ -56,30 +66,42 @@ EMAIL_SENDER_NAME=admin
 EMAIL_SENDER_DOMAIN=
 
 # Query timeouts
-SEARCH_TIMEOUT=15
+SEARCH_TIMEOUT=5
 QUERY_TIMEOUT=5
 
 # Thumbnails Generation
-ENABLE_THUMBNAIL_GENERATION=false
+ENABLE_THUMBNAIL_GENERATION=true
 
 # S3 configuration
 USE_S3=false
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+# seconds for signed S3 urls to expire
+# this is currently only used for user export files
+S3_SIGNED_URL_EXPIRY=900
 
 # Commented are example values if you use a non-AWS, S3-compatible service
 # AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
 # non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
-# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL.
+# AWS_S3_URL_PROTOCOL must end in ":" and defaults to the same protocol as
+# the BookWyrm instance ("http:" or "https:", based on USE_SSL).
 
 # AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
 # AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_URL_PROTOCOL=None        # "http:"
 # AWS_S3_REGION_NAME=None         # "fr-par"
 # AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
 
+# Commented are example values if you use Azure Blob Storage
+# USE_AZURE=true
+# AZURE_ACCOUNT_NAME=     # "example-account-name"
+# AZURE_ACCOUNT_KEY=      # "base64-encoded-access-key"
+# AZURE_CONTAINER=        # "example-blob-container-name"
+# AZURE_CUSTOM_DOMAIN=    # "example-account-name.blob.core.windows.net"
 
 # Preview image generation can be computing and storage intensive
-# ENABLE_PREVIEW_IMAGES=True
+ENABLE_PREVIEW_IMAGES=False
 
 # Specify RGB tuple or RGB hex strings,
 # or use_dominant_color_light / use_dominant_color_dark
@@ -108,3 +130,28 @@ OTEL_EXPORTER_OTLP_ENDPOINT=
 OTEL_EXPORTER_OTLP_HEADERS=
 # Service name to identify your app
 OTEL_SERVICE_NAME=
+
+# Set HTTP_X_FORWARDED_PROTO ONLY to true if you know what you are doing.
+# Only use it if your proxy is "swallowing" if the original request was made
+# via https. Please refer to the Django-Documentation and assess the risks
+# for your instance:
+# https://docs.djangoproject.com/en/3.2/ref/settings/#secure-proxy-ssl-header
+HTTP_X_FORWARDED_PROTO=false
+
+# TOTP settings
+# TWO_FACTOR_LOGIN_VALIDITY_WINDOW sets the number of codes either side
+# which will be accepted.
+TWO_FACTOR_LOGIN_VALIDITY_WINDOW=2
+TWO_FACTOR_LOGIN_MAX_SECONDS=60
+
+# Additional hosts to allow in the Content-Security-Policy, "self" (should be
+# DOMAIN with optionally ":" + PORT) and AWS_S3_CUSTOM_DOMAIN (if used) are
+# added by default.  Value should be a comma-separated list of host names.
+CSP_ADDITIONAL_HOSTS=
+
+# Time before being logged out (in seconds)
+# SESSION_COOKIE_AGE=2592000   # current default: 30 days
+
+# Maximum allowed memory for file uploads (increase if users are having trouble
+# uploading BookWyrm export files).
+# DATA_UPLOAD_MAX_MEMORY_MiB=100

.github/workflows/black.yml

@@ -1,15 +0,0 @@
-name: Python Formatting (run ./bw-dev black to fix)
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - uses: psf/black@21.4b2

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,4 +65,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.github/workflows/curlylint.yaml

@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install curlylint
         run: pip install curlylint

.github/workflows/django-tests.yml

@@ -1,60 +0,0 @@
-name: Run Python Tests
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-20.04
-    services:
-      postgres:
-        image: postgres:13
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: hunter2
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Run Tests
-      env:
-        SECRET_KEY: beepbeep
-        DEBUG: false
-        USE_HTTPS: true
-        DOMAIN: your.domain.here
-        BOOKWYRM_DATABASE_BACKEND: postgres
-        MEDIA_ROOT: images/
-        POSTGRES_PASSWORD: hunter2
-        POSTGRES_USER: postgres
-        POSTGRES_DB: github_actions
-        POSTGRES_HOST: 127.0.0.1
-        CELERY_BROKER: ""
-        REDIS_BROKER_PORT: 6379
-        REDIS_BROKER_PASSWORD: beep
-        USE_DUMMY_CACHE: true
-        FLOWER_PORT: 8888
-        EMAIL_HOST: "smtp.mailgun.org"
-        EMAIL_PORT: 587
-        EMAIL_HOST_USER: ""
-        EMAIL_HOST_PASSWORD: ""
-        EMAIL_USE_TLS: true
-        ENABLE_PREVIEW_IMAGES: false
-        ENABLE_THUMBNAIL_GENERATION: true
-      run: |
-        pytest -n 3

.github/workflows/lint-frontend.yaml

@@ -19,16 +19,17 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install modules
-        run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+      #  run: npm install stylelint stylelint-config-recommended stylelint-config-standard stylelint-order eslint
+        run: npm install eslint@^8.9.0
 
       # See .stylelintignore for files that are not linted.
-      - name: Run stylelint
-        run: >
-          npx stylelint bookwyrm/static/css/*.scss bookwyrm/static/css/bookwyrm/**/*.scss \
-            --config dev-tools/.stylelintrc.js
+      # - name: Run stylelint
+      #   run: >
+      #     npx stylelint bookwyrm/static/css/*.scss bookwyrm/static/css/bookwyrm/**/*.scss \
+      #       --config dev-tools/.stylelintrc.js
 
       # See .eslintignore for files that are not linted.
       - name: Run ESLint

.github/workflows/prettier.yaml

@@ -14,10 +14,10 @@ jobs:
 
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install modules
-        run: npm install prettier
+        run: npm install prettier@2.5.1
 
       - name: Run Prettier
         run: npx prettier --check bookwyrm/static/js/*.js

.github/workflows/pylint.yml

@@ -1,27 +0,0 @@
-name: Pylint
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
-    - name: Install Dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Analysing the code with pylint
-      run: |
-        pylint bookwyrm/
-

.github/workflows/python.yml

@@ -0,0 +1,99 @@
+name: Python
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# overrides for .env.example
+env:
+  POSTGRES_HOST: 127.0.0.1
+  PGPORT: 5432
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: hunter2
+  POSTGRES_DB: github_actions
+  SECRET_KEY: beepbeep
+  EMAIL_HOST_USER: ""
+  EMAIL_HOST_PASSWORD: ""
+
+jobs:
+  pytest:
+    name: Tests (pytest)
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:13
+        env: # does not inherit from jobs.build.env
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: hunter2
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest-github-actions-annotate-failures
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Check migrations up-to-date
+      run: python ./manage.py makemigrations --check
+    - name: Run Tests
+      run: pytest -n 3
+
+  pylint:
+    name: Linting (pylint)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Analyse code with pylint
+      run: pylint bookwyrm/
+
+  mypy:
+    name: Typing (mypy)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: pip
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Set up .env
+      run: cp .env.example .env
+    - name: Analyse code with mypy
+      run: mypy bookwyrm celerywyrm
+
+  black:
+    name: Formatting (black; run ./bw-dev black to fix)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+    - uses: psf/black@stable
+      with:
+        version: "22.*"

.gitignore

@@ -16,6 +16,8 @@
 # BookWyrm
 .env
 /images/
+/exports/
+/static/
 bookwyrm/static/css/bookwyrm.css
 bookwyrm/static/css/themes/
 !bookwyrm/static/css/themes/bookwyrm-*.scss
@@ -36,3 +38,6 @@ nginx/default.conf
 
 #macOS
 **/.DS_Store
+
+# Docker
+docker-compose.override.yml

.prettierrc

@@ -0,0 +1 @@
+'trailingComma': 'es5'

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11
 
 ENV PYTHONUNBUFFERED 1
 

FEDERATION.md

@@ -0,0 +1,334 @@
+# Federation
+
+BookWyrm uses the [ActivityPub](http://activitypub.rocks/) protocol to send and receive user activity between other BookWyrm instances and other services that implement ActivityPub. To handle book data, BookWyrm has a handful of extended Activity types which are not part of the standard, but are legible to other BookWyrm instances.
+
+## Activities and Objects
+
+### Users and relationships
+User relationship interactions follow the standard ActivityPub spec.
+
+- `Follow`: request to receive statuses from a user, and view their statuses that have followers-only privacy
+- `Accept`: approves a `Follow` and finalizes the relationship
+- `Reject`: denies a `Follow`
+- `Block`: prevent users from seeing one another's statuses, and prevents the blocked user from viewing the actor's profile
+- `Update`: updates a user's profile and settings
+- `Delete`: deactivates a user
+- `Undo`: reverses a `Block` or `Follow`
+
+### Activities
+- `Create/Status`: saves a new status in the database.
+- `Delete/Status`: Removes a status
+- `Like/Status`: Creates a favorite on the status
+- `Announce/Status`: Boosts the status into the actor's timeline
+- `Undo/*`,: Reverses an `Announce`, `Like`, or `Move`
+- `Move/User`: Moves a user from one ActivityPub id to another.
+
+### Collections
+User's books and lists are represented by [`OrderedCollection`](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-orderedcollection)
+
+### Statuses
+
+BookWyrm is focused on book reading activities - it is not a general-purpose messaging application. For this reason, BookWyrm only accepts status `Create` activities if they are:
+
+- Direct messages (i.e., `Note`s with the privacy level `direct`, which mention a local user),
+- Related to a book (of a custom status type that includes the field `inReplyToBook`),
+- Replies to existing statuses saved in the database
+
+All other statuses will be received by the instance inbox, but by design **will not be delivered to user inboxes or displayed to users**.
+
+### Custom Object types
+
+With the exception of `Note`, the following object types are used in Bookwyrm but are not currently provided with a custom JSON-LD `@context` extension IRI. This is likely to change in future to make them true deserialisable JSON-LD objects.
+
+##### Note
+
+Within BookWyrm a `Note` is constructed according to [the ActivityStreams vocabulary](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-note), however `Note`s can only be created as direct messages or as replies to other statuses. As mentioned above, this also applies to incoming `Note`s.
+
+##### Review
+
+A `Review` is a status in response to a book (indicated by the `inReplyToBook` field), which has a title, body, and numerical rating between 0 (not rated) and 5.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/library_lurker/review/2",
+    "type": "Review",
+    "published": "2023-06-30T21:43:46.013132+00:00",
+    "attributedTo": "https://example.net/user/library_lurker",
+    "content": "<p>This is an enjoyable book with great characters.</p>",
+    "to": ["https://example.net/user/library_lurker/followers"],
+    "cc": [],
+    "replies": {
+        "id": "https://example.net/user/library_lurker/review/2/replies",
+        "type": "OrderedCollection",
+        "totalItems": 0,
+        "first": "https://example.net/user/library_lurker/review/2/replies?page=1",
+        "last": "https://example.net/user/library_lurker/review/2/replies?page=1",
+        "@context": "https://www.w3.org/ns/activitystreams"
+        },
+        "summary": "Spoilers ahead!",
+        "tag": [],
+        "attachment": [],
+        "sensitive": true,
+    "inReplyToBook": "https://example.net/book/1",
+    "name": "What a cracking read",
+    "rating": 4.5,
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Comment
+
+A `Comment` on a book mentions a book and has a message body, reading status, and progress indicator.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/library_lurker/comment/9",
+    "type": "Comment",
+    "published": "2023-06-30T21:43:46.013132+00:00",
+    "attributedTo": "https://example.net/user/library_lurker",
+    "content": "<p>This is a very enjoyable book so far.</p>",
+    "to": ["https://example.net/user/library_lurker/followers"],
+    "cc": [],
+    "replies": {
+        "id": "https://example.net/user/library_lurker/comment/9/replies",
+        "type": "OrderedCollection",
+        "totalItems": 0,
+        "first": "https://example.net/user/library_lurker/comment/9/replies?page=1",
+        "last": "https://example.net/user/library_lurker/comment/9/replies?page=1",
+        "@context": "https://www.w3.org/ns/activitystreams"
+        },
+        "summary": "Spoilers ahead!",
+        "tag": [],
+        "attachment": [],
+        "sensitive": true,
+    "inReplyToBook": "https://example.net/book/1",
+    "readingStatus": "reading",
+    "progress": 25,
+    "progressMode": "PG",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Quotation
+
+A quotation (aka "quote") has a message body, an excerpt from a book including position as a page number or percentage indicator, and mentions a book.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/user/mouse/quotation/13",
+    "url": "https://example.net/user/mouse/quotation/13",
+    "inReplyTo": null,
+    "published": "2020-05-10T02:38:31.150343+00:00",
+    "attributedTo": "https://example.net/user/mouse",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+        ],
+    "cc": [
+        "https://example.net/user/mouse/followers"
+        ],
+    "sensitive": false,
+    "content": "I really like this quote",
+    "type": "Quotation",
+    "replies": {
+        "id": "https://example.net/user/mouse/quotation/13/replies",
+        "type": "Collection",
+        "first": {
+            "type": "CollectionPage",
+            "next": "https://example.net/user/mouse/quotation/13/replies?only_other_accounts=true&page=true",
+            "partOf": "https://example.net/user/mouse/quotation/13/replies",
+            "items": []
+            }
+        },
+    "inReplyToBook": "https://example.net/book/1",
+    "quote": "To be or not to be, that is the question.",
+    "position": 50,
+    "positionMode": "PCT",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+### Custom Objects
+
+##### Work
+A particular book, a "work" in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
+
+Example:
+
+```json
+{
+    "id": "https://bookwyrm.social/book/5988",
+    "type": "Work",
+    "authors": [
+        "https://bookwyrm.social/author/417"
+    ],
+    "first_published_date": null,
+    "published_date": null,
+    "title": "Piranesi",
+    "sort_title": null,
+    "subtitle": null,
+    "description": "**From the *New York Times* bestselling author of *Jonathan Strange & Mr. Norrell*, an intoxicating, hypnotic new novel set in a dreamlike alternative reality.",
+    "languages": [],
+    "series": null,
+    "series_number": null,
+    "subjects": [
+        "English literature"
+    ],
+    "subject_places": [],
+    "openlibrary_key": "OL20893680W",
+    "librarything_key": null,
+    "goodreads_key": null,
+    "attachment": [
+        {
+            "url": "https://bookwyrm.social/images/covers/10226290-M.jpg",
+            "type": "Image"
+        }
+    ],
+    "lccn": null,
+    "editions": [
+        "https://bookwyrm.social/book/5989"
+    ],
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+##### Edition
+A particular _manifestation_ of a Work, in the [FRBR](https://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records) sense.
+
+Example:
+
+```json
+{
+    "id": "https://bookwyrm.social/book/5989",
+    "lastEditedBy": "https://example.net/users/rat",
+    "type": "Edition",
+    "authors": [
+        "https://bookwyrm.social/author/417"
+    ],
+    "first_published_date": null,
+    "published_date": "2020-09-15T00:00:00+00:00",
+    "title": "Piranesi",
+    "sort_title": null,
+    "subtitle": null,
+    "description": "Piranesi's house is no ordinary building; its rooms are infinite, its corridors endless, its walls are lined with thousands upon thousands of statues, each one different from all the others.",
+    "languages": [
+        "English"
+    ],
+    "series": null,
+    "series_number": null,
+    "subjects": [],
+    "subject_places": [],
+    "openlibrary_key": "OL29486417M",
+    "librarything_key": null,
+    "goodreads_key": null,
+    "isfdb": null,
+    "attachment": [
+        {
+            "url": "https://bookwyrm.social/images/covers/50202953._SX318_.jpg",
+            "type": "Image"
+        }
+    ],
+    "isbn_10": "1526622424",
+    "isbn_13": "9781526622426",
+    "oclc_number": null,
+    "asin": null,
+    "pages": 272,
+    "physical_format": null,
+    "publishers": [
+        "Bloomsbury Publishing Plc"
+    ],
+    "work": "https://bookwyrm.social/book/5988",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### Shelf
+
+A user's book collection. By default, every user has a `to-read`, `reading`, `read`, and `stopped-reading` shelf which are used to track reading progress. Users may create an unlimited number of additional shelves with their own ids.
+
+Example
+
+```json
+{
+    "id": "https://example.net/user/avid_reader/books/extraspecialbooks-5",
+    "type": "Shelf",
+    "totalItems": 0,
+    "first": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
+    "last": "https://example.net/user/avid_reader/books/extraspecialbooks-5?page=1",
+    "name": "Extra special books",
+    "owner": "https://example.net/user/avid_reader",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+        "https://example.net/user/avid_reader/followers"
+    ],
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### List
+
+A collection of books that may have items contributed by users other than the one who created the list.
+
+Example:
+
+```json
+{
+    "id": "https://example.net/list/1",
+    "type": "BookList",
+    "totalItems": 0,
+    "first": "https://example.net/list/1?page=1",
+    "last": "https://example.net/list/1?page=1",
+    "name": "My cool list",
+    "owner": "https://example.net/user/avid_reader",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+        "https://example.net/user/avid_reader/followers"
+    ],
+    "summary": "A list of books I like.",
+    "curation": "curated",
+    "@context": "https://www.w3.org/ns/activitystreams"
+}
+```
+
+#### Activities
+
+- `Create`: Adds a shelf or list to the database.
+- `Delete`: Removes a shelf or list.
+- `Add`: Adds a book to a shelf or list.
+- `Remove`: Removes a book from a shelf or list.
+
+## Alternative Serialization
+Because BookWyrm uses custom object types that aren't listed in [the standard ActivityStreams Vocabulary](https://www.w3.org/TR/activitystreams-vocabulary), some statuses are transformed into standard types when sent to or viewed by non-BookWyrm services. `Review`s are converted into `Article`s, and `Comment`s and `Quotation`s are converted into `Note`s, with a link to the book and the cover image attached.
+
+In future this may be done with [JSON-LD type arrays](https://www.w3.org/TR/json-ld/#specifying-the-type) instead.
+
+## Other extensions
+
+### Webfinger
+
+Bookwyrm uses the [Webfinger](https://datatracker.ietf.org/doc/html/rfc7033) standard to identify and disambiguate fediverse actors. The [Webfinger documentation on the Mastodon project](https://docs.joinmastodon.org/spec/webfinger/) provides a good overview of how Webfinger is used.
+
+### HTTP Signatures
+
+Bookwyrm uses and requires HTTP signatures for all `POST` requests. `GET` requests are not signed by default, but if Bookwyrm receives a `403` response to a `GET` it will re-send the request, signed by the default server user. This usually will have a user id of `https://example.net/user/bookwyrm.instance.actor`
+
+#### publicKey id
+
+In older versions of Bookwyrm the `publicKey.id` was incorrectly listed in request headers as `https://example.net/user/username#main-key`. As of v0.6.3 the id is now listed correctly, as `https://example.net/user/username/#main-key`. In most ActivityPub implementations this will make no difference as the URL will usually resolve to the same place.
+
+### NodeInfo
+
+Bookwyrm uses the [NodeInfo](http://nodeinfo.diaspora.software/) standard to provide statistics and version information for each instance.
+
+## Further Documentation
+
+See [docs.joinbookwyrm.com/](https://docs.joinbookwyrm.com/) for more documentation.

README.md

@@ -1,60 +1,44 @@
 # BookWyrm
 
-Social reading and reviewing, decentralized with ActivityPub
+[![](https://img.shields.io/github/release/bookwyrm-social/bookwyrm.svg?colorB=58839b)](https://github.com/bookwyrm-social/bookwyrm/releases)
+[![Run Python Tests](https://github.com/bookwyrm-social/bookwyrm/actions/workflows/django-tests.yml/badge.svg)](https://github.com/bookwyrm-social/bookwyrm/actions/workflows/django-tests.yml)
+[![Pylint](https://github.com/bookwyrm-social/bookwyrm/actions/workflows/pylint.yml/badge.svg)](https://github.com/bookwyrm-social/bookwyrm/actions/workflows/pylint.yml)
 
-## Contents
-- [Joining BookWyrm](#joining-bookwyrm)
-- [Contributing](#contributing)
-- [About BookWyrm](#about-bookwyrm)
-    - [What it is and isn't](#what-it-is-and-isnt)
-    - [The role of federation](#the-role-of-federation)
-    - [Features](#features)
-- [Set up BookWyrm](#set-up-bookwyrm)
-
-## Joining BookWyrm
-If you'd like to join an instance, you can check out the [instances](https://joinbookwyrm.com/instances/) list.
+BookWyrm is a social network for tracking your reading, talking about books, writing reviews, and discovering what to read next. Federation allows BookWyrm users to join small, trusted communities that can connect with one another, and with other ActivityPub services like [Mastodon](https://joinmastodon.org/) and [Pleroma](http://pleroma.social/).
 
 
-## Contributing
-See [contributing](https://docs.joinbookwyrm.com/contributing.html) for code, translation or monetary contributions.
+## Links
+
+[![Mastodon Follow](https://img.shields.io/mastodon/follow/000146121?domain=https%3A%2F%2Ftech.lgbt&style=social)](https://tech.lgbt/@bookwyrm)
+
+ - [Project homepage](https://joinbookwyrm.com/)
+ - [Support](https://patreon.com/bookwyrm)
+ - [Documentation](https://docs.joinbookwyrm.com/)
+
 
 ## About BookWyrm
-### What it is and isn't
 BookWyrm is a platform for social reading. You can use it to track what you're reading, review books, and follow your friends. It isn't primarily meant for cataloguing or as a data-source for books, but it does do both of those things to some degree.
 
-### The role of federation
+## Federation
 BookWyrm is built on [ActivityPub](http://activitypub.rocks/). With ActivityPub, it inter-operates with different instances of BookWyrm, and other ActivityPub compliant services, like Mastodon. This means you can run an instance for your book club, and still follow your friend who posts on a server devoted to 20th century Russian speculative fiction. It also means that your friend on mastodon can read and comment on a book review that you post on your BookWyrm instance.
 
 Federation makes it possible to have small, self-determining communities, in contrast to the monolithic service you find on GoodReads or Twitter. An instance can be focused on a particular interest, be just for a group of friends, or anything else that brings people together. Each community can choose which other instances they want to federate with, and moderate and run their community autonomously. Check out https://runyourown.social/ to get a sense of the philosophy and logistics behind small, high-trust social networks.
 
-### Features
-Since the project is still in its early stages, the features are growing every day, and there is plenty of room for suggestions and ideas. Open an [issue](https://github.com/bookwyrm-social/bookwyrm/issues) to get the conversation going!
-- Posting about books
-    - Compose reviews, with or without ratings, which are aggregated in the book page
-    - Compose other kinds of statuses about books, such as:
-        - Comments on a book
-        - Quotes or excerpts
-    - Reply to statuses
-    - View aggregate reviews of a book across connected BookWyrm instances
-    - Differentiate local and federated reviews and rating in your activity feed
-- Track reading activity
-    - Shelve books on default "to-read," "currently reading," and "read" shelves
-    - Create custom shelves
-    - Store started reading/finished reading dates, as well as progress updates along the way
-    - Update followers about reading activity (optionally, and with granular privacy controls)
-    - Create lists of books which can be open to submissions from anyone, curated, or only edited by the creator
-- Federation with ActivityPub
-    - Broadcast and receive user statuses and activity
-    - Share book data between instances to create a networked database of metadata
-    - Identify shared books across instances and aggregate related content
-    - Follow and interact with users across BookWyrm instances
-    - Inter-operate with non-BookWyrm ActivityPub services (currently, Mastodon is supported)
-- Granular privacy controls
-    - Private, followers-only, and public privacy levels for posting, shelves, and lists
-    - Option for users to manually approve followers
-    - Allow blocking and flagging for moderation
+## Features
 
-### The Tech Stack
+### Post about books
+Compose reviews, comment on what you're reading, and post quotes from books. You can converse with other BookWyrm users across the network about what they're reading.
+
+### Track reading activity
+Keep track of what books you've read, and what books you'd like to read in the future.
+
+### Federation with ActivityPub
+Federation allows you to interact with users on other instances and services, and also shares metadata about books and authors, which collaboratively builds a decentralized database of books.
+
+### Privacy and moderation
+Users and administrators can control who can see their posts and what other instances to federate with.
+
+## Tech Stack
 Web backend
 - [Django](https://www.djangoproject.com/) web server
 - [PostgreSQL](https://www.postgresql.org/) database

VERSION

@@ -0,0 +1 @@
+0.7.3

bookwyrm/activitypub/__init__.py

@@ -3,8 +3,12 @@ import inspect
 import sys
 
 from .base_activity import ActivityEncoder, Signature, naive_parse
-from .base_activity import Link, Mention
-from .base_activity import ActivitySerializerError, resolve_remote_id
+from .base_activity import Link, Mention, Hashtag
+from .base_activity import (
+    ActivitySerializerError,
+    resolve_remote_id,
+    get_representative,
+)
 from .image import Document, Image
 from .note import Note, GeneratedNote, Article, Comment, Quotation
 from .note import Review, Rating
@@ -19,6 +23,7 @@ from .verbs import Create, Delete, Undo, Update
 from .verbs import Follow, Accept, Reject, Block
 from .verbs import Add, Remove
 from .verbs import Announce, Like
+from .verbs import Move
 
 # this creates a list of all the Activity types that we can serialize,
 # so when an Activity comes in from outside, we can check if it's known

bookwyrm/activitypub/base_activity.py

@@ -1,16 +1,28 @@
 """ basics for an activitypub serializer """
+from __future__ import annotations
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 import logging
+from typing import Optional, Union, TypeVar, overload, Any
+
+import requests
 
 from django.apps import apps
 from django.db import IntegrityError, transaction
+from django.utils.http import http_date
 
+from bookwyrm import models
 from bookwyrm.connectors import ConnectorException, get_data
-from bookwyrm.tasks import app
+from bookwyrm.models import base_model
+from bookwyrm.signatures import make_signature
+from bookwyrm.settings import DOMAIN, INSTANCE_ACTOR_USERNAME
+from bookwyrm.tasks import app, MISC
 
 logger = logging.getLogger(__name__)
 
+# pylint: disable=invalid-name
+TBookWyrmModel = TypeVar("TBookWyrmModel", bound=base_model.BookWyrmModel)
+
 
 class ActivitySerializerError(ValueError):
     """routine problems serializing activitypub json"""
@@ -60,7 +72,13 @@ class ActivityObject:
     id: str
     type: str
 
-    def __init__(self, activity_objects=None, **kwargs):
+    def __init__(
+        self,
+        activity_objects: Optional[
+            dict[str, Union[str, list[str], ActivityObject, base_model.BookWyrmModel]]
+        ] = None,
+        **kwargs: Any,
+    ):
         """this lets you pass in an object with fields that aren't in the
         dataclass, which it ignores. Any field in the dataclass is required or
         has a default value"""
@@ -95,16 +113,34 @@ class ActivityObject:
 
     # pylint: disable=too-many-locals,too-many-branches,too-many-arguments
     def to_model(
-        self, model=None, instance=None, allow_create=True, save=True, overwrite=True
-    ):
-        """convert from an activity to a model instance"""
+        self,
+        model: Optional[type[TBookWyrmModel]] = None,
+        instance: Optional[TBookWyrmModel] = None,
+        allow_create: bool = True,
+        save: bool = True,
+        overwrite: bool = True,
+        allow_external_connections: bool = True,
+    ) -> Optional[TBookWyrmModel]:
+        """convert from an activity to a model instance. Args:
+        model: the django model that this object is being converted to
+            (will guess if not known)
+        instance: an existing database entry that is going to be updated by
+            this activity
+        allow_create: whether a new object should be created if there is no
+            existing object is provided or found matching the remote_id
+        save: store in the database if true, return an unsaved model obj if false
+        overwrite: replace fields in the database with this activity if true,
+            only update blank fields if false
+        allow_external_connections: look up missing data if true,
+            throw an exception if false and an external connection is needed
+        """
         model = model or get_model_from_type(self.type)
 
         # only reject statuses if we're potentially creating them
         if (
             allow_create
             and hasattr(model, "ignore_activity")
-            and model.ignore_activity(self)
+            and model.ignore_activity(self, allow_external_connections)
         ):
             return None
 
@@ -122,7 +158,10 @@ class ActivityObject:
         for field in instance.simple_fields:
             try:
                 changed = field.set_field_from_activity(
-                    instance, self, overwrite=overwrite
+                    instance,
+                    self,
+                    overwrite=overwrite,
+                    allow_external_connections=allow_external_connections,
                 )
                 if changed:
                     update_fields.append(field.name)
@@ -133,7 +172,11 @@ class ActivityObject:
         # too early and jank up users
         for field in instance.image_fields:
             changed = field.set_field_from_activity(
-                instance, self, save=save, overwrite=overwrite
+                instance,
+                self,
+                save=save,
+                overwrite=overwrite,
+                allow_external_connections=allow_external_connections,
             )
             if changed:
                 update_fields.append(field.name)
@@ -156,8 +199,12 @@ class ActivityObject:
 
             # add many to many fields, which have to be set post-save
             for field in instance.many_to_many_fields:
-                # mention books/users, for example
-                field.set_field_from_activity(instance, self)
+                # mention books/users/hashtags, for example
+                field.set_field_from_activity(
+                    instance,
+                    self,
+                    allow_external_connections=allow_external_connections,
+                )
 
         # reversed relationships in the models
         for (
@@ -190,10 +237,15 @@ class ActivityObject:
         omit = kwargs.get("omit", ())
         data = self.__dict__.copy()
         # recursively serialize
-        for (k, v) in data.items():
+        for k, v in data.items():
             try:
                 if issubclass(type(v), ActivityObject):
                     data[k] = v.serialize()
+                elif isinstance(v, list):
+                    data[k] = [
+                        e.serialize() if issubclass(type(e), ActivityObject) else e
+                        for e in v
+                    ]
             except TypeError:
                 pass
         data = {k: v for (k, v) in data.items() if v is not None and k not in omit}
@@ -202,7 +254,7 @@ class ActivityObject:
         return data
 
 
-@app.task(queue="medium_priority")
+@app.task(queue=MISC)
 @transaction.atomic
 def set_related_field(
     model_name, origin_model_name, related_field_name, related_remote_id, data
@@ -241,10 +293,10 @@ def set_related_field(
 
 def get_model_from_type(activity_type):
     """given the activity, what type of model"""
-    models = apps.get_models()
+    activity_models = apps.get_models()
     model = [
         m
-        for m in models
+        for m in activity_models
         if hasattr(m, "activity_serializer")
         and hasattr(m.activity_serializer, "type")
         and m.activity_serializer.type == activity_type
@@ -256,10 +308,48 @@ def get_model_from_type(activity_type):
     return model[0]
 
 
+# pylint: disable=too-many-arguments
+@overload
 def resolve_remote_id(
-    remote_id, model=None, refresh=False, save=True, get_activity=False
-):
-    """take a remote_id and return an instance, creating if necessary"""
+    remote_id: str,
+    model: type[TBookWyrmModel],
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> TBookWyrmModel:
+    ...
+
+
+# pylint: disable=too-many-arguments
+@overload
+def resolve_remote_id(
+    remote_id: str,
+    model: Optional[str] = None,
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> base_model.BookWyrmModel:
+    ...
+
+
+# pylint: disable=too-many-arguments
+def resolve_remote_id(
+    remote_id: str,
+    model: Optional[Union[str, type[base_model.BookWyrmModel]]] = None,
+    refresh: bool = False,
+    save: bool = True,
+    get_activity: bool = False,
+    allow_external_connections: bool = True,
+) -> base_model.BookWyrmModel:
+    """take a remote_id and return an instance, creating if necessary. Args:
+    remote_id: the unique url for looking up the object in the db or by http
+    model: a string or object representing the model that corresponds to the object
+    save: whether to return an unsaved database entry or a saved one
+    get_activity: whether to return the activitypub object or the model object
+    allow_external_connections: whether to make http connections
+    """
     if model:  # a bonus check we can do if we already know the model
         if isinstance(model, str):
             model = apps.get_model(f"bookwyrm.{model}", require_ready=True)
@@ -267,13 +357,26 @@ def resolve_remote_id(
         if result and not refresh:
             return result if not get_activity else result.to_activity_dataclass()
 
+    # The above block will return the object if it already exists in the database.
+    # If it doesn't, an external connection would be needed, so check if that's cool
+    if not allow_external_connections:
+        raise ActivitySerializerError(
+            "Unable to serialize object without making external HTTP requests"
+        )
+
     # load the data and create the object
     try:
         data = get_data(remote_id)
-    except ConnectorException:
-        logger.exception("Could not connect to host for remote_id: %s", remote_id)
+    except ConnectionError:
+        logger.info("Could not connect to host for remote_id: %s", remote_id)
         return None
-
+    except requests.HTTPError as e:
+        if (e.response is not None) and e.response.status_code == 401:
+            # This most likely means it's a mastodon with secure fetch enabled.
+            data = get_activitypub_data(remote_id)
+        else:
+            logger.info("Could not connect to host for remote_id: %s", remote_id)
+            return None
     # determine the model implicitly, if not provided
     # or if it's a model with subclasses like Status, check again
     if not model or hasattr(model.objects, "select_subclasses"):
@@ -292,6 +395,49 @@ def resolve_remote_id(
     return item.to_model(model=model, instance=result, save=save)
 
 
+def get_representative():
+    """Get or create an actor representing the instance
+    to sign outgoing HTTP GET requests"""
+    return models.User.objects.get_or_create(
+        username=f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}",
+        defaults=dict(
+            email="bookwyrm@localhost",
+            local=True,
+            localname=INSTANCE_ACTOR_USERNAME,
+        ),
+    )[0]
+
+
+def get_activitypub_data(url):
+    """wrapper for request.get"""
+    now = http_date()
+    sender = get_representative()
+    if not sender.key_pair.private_key:
+        # this shouldn't happen. it would be bad if it happened.
+        raise ValueError("No private key found for sender")
+    try:
+        resp = requests.get(
+            url,
+            headers={
+                # pylint: disable=line-too-long
+                "Accept": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+                "Date": now,
+                "Signature": make_signature("get", sender, url, now),
+            },
+            timeout=15,
+        )
+    except requests.RequestException:
+        raise ConnectorException()
+    if not resp.ok:
+        resp.raise_for_status()
+    try:
+        data = resp.json()
+    except ValueError:
+        raise ConnectorException()
+
+    return data
+
+
 @dataclass(init=False)
 class Link(ActivityObject):
     """for tagging a book in a status"""
@@ -306,7 +452,9 @@ class Link(ActivityObject):
 
     def serialize(self, **kwargs):
         """remove fields"""
-        omit = ("id", "type", "@context")
+        omit = ("id", "@context")
+        if self.type == "Link":
+            omit += ("type",)
         return super().serialize(omit=omit)
 
 
@@ -315,3 +463,10 @@ class Mention(Link):
     """a subtype of Link for mentioning an actor"""
 
     type: str = "Mention"
+
+
+@dataclass(init=False)
+class Hashtag(Link):
+    """a subtype of Link for mentioning a hashtag"""
+
+    type: str = "Hashtag"

bookwyrm/activitypub/book.py

@@ -1,6 +1,6 @@
 """ book and author data """
 from dataclasses import dataclass, field
-from typing import List
+from typing import Optional
 
 from .base_activity import ActivityObject
 from .image import Document
@@ -11,17 +11,17 @@ from .image import Document
 class BookData(ActivityObject):
     """shared fields for all book data and authors"""
 
-    openlibraryKey: str = None
-    inventaireId: str = None
-    librarythingKey: str = None
-    goodreadsKey: str = None
-    bnfId: str = None
-    viaf: str = None
-    wikidata: str = None
-    asin: str = None
-    lastEditedBy: str = None
-    links: List[str] = field(default_factory=lambda: [])
-    fileLinks: List[str] = field(default_factory=lambda: [])
+    openlibraryKey: Optional[str] = None
+    inventaireId: Optional[str] = None
+    librarythingKey: Optional[str] = None
+    goodreadsKey: Optional[str] = None
+    bnfId: Optional[str] = None
+    viaf: Optional[str] = None
+    wikidata: Optional[str] = None
+    asin: Optional[str] = None
+    aasin: Optional[str] = None
+    isfdb: Optional[str] = None
+    lastEditedBy: Optional[str] = None
 
 
 # pylint: disable=invalid-name
@@ -33,17 +33,19 @@ class Book(BookData):
     sortTitle: str = None
     subtitle: str = None
     description: str = ""
-    languages: List[str] = field(default_factory=lambda: [])
+    languages: list[str] = field(default_factory=list)
     series: str = ""
     seriesNumber: str = ""
-    subjects: List[str] = field(default_factory=lambda: [])
-    subjectPlaces: List[str] = field(default_factory=lambda: [])
+    subjects: list[str] = field(default_factory=list)
+    subjectPlaces: list[str] = field(default_factory=list)
 
-    authors: List[str] = field(default_factory=lambda: [])
+    authors: list[str] = field(default_factory=list)
     firstPublishedDate: str = ""
     publishedDate: str = ""
 
-    cover: Document = None
+    fileLinks: list[str] = field(default_factory=list)
+
+    cover: Optional[Document] = None
     type: str = "Book"
 
 
@@ -56,10 +58,10 @@ class Edition(Book):
     isbn10: str = ""
     isbn13: str = ""
     oclcNumber: str = ""
-    pages: int = None
+    pages: Optional[int] = None
     physicalFormat: str = ""
     physicalFormatDetail: str = ""
-    publishers: List[str] = field(default_factory=lambda: [])
+    publishers: list[str] = field(default_factory=list)
     editionRank: int = 0
 
     type: str = "Edition"
@@ -71,7 +73,7 @@ class Work(Book):
     """work instance of a book object"""
 
     lccn: str = ""
-    editions: List[str] = field(default_factory=lambda: [])
+    editions: list[str] = field(default_factory=list)
     type: str = "Work"
 
 
@@ -81,12 +83,13 @@ class Author(BookData):
     """author of a book"""
 
     name: str
-    isni: str = None
-    viafId: str = None
-    gutenbergId: str = None
-    born: str = None
-    died: str = None
-    aliases: List[str] = field(default_factory=lambda: [])
+    isni: Optional[str] = None
+    viafId: Optional[str] = None
+    gutenbergId: Optional[str] = None
+    born: Optional[str] = None
+    died: Optional[str] = None
+    aliases: list[str] = field(default_factory=list)
     bio: str = ""
     wikipediaLink: str = ""
     type: str = "Author"
+    website: str = ""

bookwyrm/activitypub/note.py

@@ -1,9 +1,12 @@
 """ note serializer and children thereof """
 from dataclasses import dataclass, field
 from typing import Dict, List
-from django.apps import apps
+import re
 
-from .base_activity import ActivityObject, Link
+from django.apps import apps
+from django.db import IntegrityError, transaction
+
+from .base_activity import ActivityObject, ActivitySerializerError, Link
 from .image import Document
 
 
@@ -38,6 +41,47 @@ class Note(ActivityObject):
     updated: str = None
     type: str = "Note"
 
+    # pylint: disable=too-many-arguments
+    def to_model(
+        self,
+        model=None,
+        instance=None,
+        allow_create=True,
+        save=True,
+        overwrite=True,
+        allow_external_connections=True,
+    ):
+        instance = super().to_model(
+            model, instance, allow_create, save, overwrite, allow_external_connections
+        )
+
+        if instance is None:
+            return instance
+
+        # Replace links to hashtags in content with local URLs
+        changed_content = False
+        for hashtag in instance.mention_hashtags.all():
+            updated_content = re.sub(
+                rf'(<a href=")[^"]*(" data-mention="hashtag">{hashtag.name}</a>)',
+                rf"\1{hashtag.remote_id}\2",
+                instance.content,
+                flags=re.IGNORECASE,
+            )
+            if instance.content != updated_content:
+                instance.content = updated_content
+                changed_content = True
+
+        if not save or not changed_content:
+            return instance
+
+        with transaction.atomic():
+            try:
+                instance.save(broadcast=False, update_fields=["content"])
+            except IntegrityError as e:
+                raise ActivitySerializerError(e)
+
+        return instance
+
 
 @dataclass(init=False)
 class Article(Note):

bookwyrm/activitypub/person.py

@@ -1,5 +1,5 @@
 """ actor serializer """
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Dict
 
 from .base_activity import ActivityObject
@@ -35,9 +35,11 @@ class Person(ActivityObject):
     endpoints: Dict = None
     name: str = None
     summary: str = None
-    icon: Image = field(default_factory=lambda: {})
+    icon: Image = None
     bookwyrmUser: bool = False
     manuallyApprovesFollowers: str = False
     discoverable: str = False
     hideFollows: str = False
+    movedTo: str = None
+    alsoKnownAs: dict[str] = None
     type: str = "Person"

bookwyrm/activitypub/verbs.py

@@ -14,12 +14,12 @@ class Verb(ActivityObject):
     actor: str
     object: ActivityObject
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """usually we just want to update and save"""
         # self.object may return None if the object is invalid in an expected way
         # ie, Question type
         if self.object:
-            self.object.to_model()
+            self.object.to_model(allow_external_connections=allow_external_connections)
 
 
 # pylint: disable=invalid-name
@@ -42,7 +42,7 @@ class Delete(Verb):
     cc: List[str] = field(default_factory=lambda: [])
     type: str = "Delete"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and delete the activity object"""
         if not self.object:
             return
@@ -52,7 +52,11 @@ class Delete(Verb):
             model = apps.get_model("bookwyrm.User")
             obj = model.find_existing_by_remote_id(self.object)
         else:
-            obj = self.object.to_model(save=False, allow_create=False)
+            obj = self.object.to_model(
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
 
         if obj:
             obj.delete()
@@ -67,11 +71,13 @@ class Update(Verb):
     to: List[str]
     type: str = "Update"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """update a model instance from the dataclass"""
         if not self.object:
             return
-        self.object.to_model(allow_create=False)
+        self.object.to_model(
+            allow_create=False, allow_external_connections=allow_external_connections
+        )
 
 
 @dataclass(init=False)
@@ -80,10 +86,10 @@ class Undo(Verb):
 
     type: str = "Undo"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and remove the activity object"""
         if isinstance(self.object, str):
-            # it may be that sometihng should be done with these, but idk what
+            # it may be that something should be done with these, but idk what
             # this seems just to be coming from pleroma
             return
 
@@ -92,13 +98,28 @@ class Undo(Verb):
         model = None
         if self.object.type == "Follow":
             model = apps.get_model("bookwyrm.UserFollows")
-            obj = self.object.to_model(model=model, save=False, allow_create=False)
+            obj = self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
             if not obj:
-                # this could be a folloq request not a follow proper
+                # this could be a follow request not a follow proper
                 model = apps.get_model("bookwyrm.UserFollowRequest")
-                obj = self.object.to_model(model=model, save=False, allow_create=False)
+                obj = self.object.to_model(
+                    model=model,
+                    save=False,
+                    allow_create=False,
+                    allow_external_connections=allow_external_connections,
+                )
         else:
-            obj = self.object.to_model(model=model, save=False, allow_create=False)
+            obj = self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            )
         if not obj:
             # if we don't have the object, we can't undo it. happens a lot with boosts
             return
@@ -112,9 +133,9 @@ class Follow(Verb):
     object: str
     type: str = "Follow"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """relationship save"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 @dataclass(init=False)
@@ -124,9 +145,9 @@ class Block(Verb):
     object: str
     type: str = "Block"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """relationship save"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 @dataclass(init=False)
@@ -136,7 +157,7 @@ class Accept(Verb):
     object: Follow
     type: str = "Accept"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """accept a request"""
         obj = self.object.to_model(save=False, allow_create=True)
         obj.accept()
@@ -149,10 +170,20 @@ class Reject(Verb):
     object: Follow
     type: str = "Reject"
 
-    def action(self):
-        """reject a follow request"""
-        obj = self.object.to_model(save=False, allow_create=False)
-        obj.reject()
+    def action(self, allow_external_connections=True):
+        """reject a follow or follow request"""
+
+        for model_name in ["UserFollowRequest", "UserFollows", None]:
+            model = apps.get_model(f"bookwyrm.{model_name}") if model_name else None
+            if obj := self.object.to_model(
+                model=model,
+                save=False,
+                allow_create=False,
+                allow_external_connections=allow_external_connections,
+            ):
+                # Reject the first model that can be built.
+                obj.reject()
+                break
 
 
 @dataclass(init=False)
@@ -163,7 +194,7 @@ class Add(Verb):
     object: CollectionItem
     type: str = "Add"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """figure out the target to assign the item to a collection"""
         target = resolve_remote_id(self.target)
         item = self.object.to_model(save=False)
@@ -177,7 +208,7 @@ class Remove(Add):
 
     type: str = "Remove"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """find and remove the activity object"""
         obj = self.object.to_model(save=False, allow_create=False)
         if obj:
@@ -191,9 +222,9 @@ class Like(Verb):
     object: str
     type: str = "Like"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """like"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
 
 
 # pylint: disable=invalid-name
@@ -207,6 +238,33 @@ class Announce(Verb):
     object: str
     type: str = "Announce"
 
-    def action(self):
+    def action(self, allow_external_connections=True):
         """boost"""
-        self.to_model()
+        self.to_model(allow_external_connections=allow_external_connections)
+
+
+@dataclass(init=False)
+class Move(Verb):
+    """a user moving an object"""
+
+    object: str
+    type: str = "Move"
+    origin: str = None
+    target: str = None
+
+    def action(self, allow_external_connections=True):
+        """move"""
+
+        object_is_user = resolve_remote_id(remote_id=self.object, model="User")
+
+        if object_is_user:
+            model = apps.get_model("bookwyrm.MoveUser")
+
+            self.to_model(
+                model=model,
+                save=True,
+                allow_external_connections=allow_external_connections,
+            )
+        else:
+            # we might do something with this to move other objects at some point
+            pass

bookwyrm/activitystreams.py

@@ -4,27 +4,32 @@ from django.dispatch import receiver
 from django.db import transaction
 from django.db.models import signals, Q
 from django.utils import timezone
+from opentelemetry import trace
 
 from bookwyrm import models
 from bookwyrm.redis_store import RedisStore, r
-from bookwyrm.tasks import app, LOW, MEDIUM, HIGH
+from bookwyrm.tasks import app, STREAMS, IMPORT_TRIGGERED
+from bookwyrm.telemetry import open_telemetry
+
+
+tracer = open_telemetry.tracer()
 
 
 class ActivityStream(RedisStore):
     """a category of activity stream (like home, local, books)"""
 
-    def stream_id(self, user):
+    def stream_id(self, user_id):
         """the redis key for this user's instance of this stream"""
-        return f"{user.id}-{self.key}"
+        return f"{user_id}-{self.key}"
 
-    def unread_id(self, user):
+    def unread_id(self, user_id):
         """the redis key for this user's unread count for this stream"""
-        stream_id = self.stream_id(user)
+        stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread"
 
-    def unread_by_status_type_id(self, user):
+    def unread_by_status_type_id(self, user_id):
         """the redis key for this user's unread count for this stream"""
-        stream_id = self.stream_id(user)
+        stream_id = self.stream_id(user_id)
         return f"{stream_id}-unread-by-type"
 
     def get_rank(self, obj):  # pylint: disable=no-self-use
@@ -33,16 +38,19 @@ class ActivityStream(RedisStore):
 
     def add_status(self, status, increment_unread=False):
         """add a status to users' feeds"""
+        audience = self.get_audience(status)
         # the pipeline contains all the add-to-stream activities
-        pipeline = self.add_object_to_related_stores(status, execute=False)
+        pipeline = self.add_object_to_stores(
+            status, self.get_stores_for_users(audience), execute=False
+        )
 
         if increment_unread:
-            for user in self.get_audience(status):
+            for user_id in audience:
                 # add to the unread status count
-                pipeline.incr(self.unread_id(user))
+                pipeline.incr(self.unread_id(user_id))
                 # add to the unread status count for status type
                 pipeline.hincrby(
-                    self.unread_by_status_type_id(user), get_status_type(status), 1
+                    self.unread_by_status_type_id(user_id), get_status_type(status), 1
                 )
 
         # and go!
@@ -52,21 +60,21 @@ class ActivityStream(RedisStore):
         """add a user's statuses to another user's feed"""
         # only add the statuses that the viewer should be able to see (ie, not dms)
         statuses = models.Status.privacy_filter(viewer).filter(user=user)
-        self.bulk_add_objects_to_store(statuses, self.stream_id(viewer))
+        self.bulk_add_objects_to_store(statuses, self.stream_id(viewer.id))
 
     def remove_user_statuses(self, viewer, user):
         """remove a user's status from another user's feed"""
         # remove all so that followers only statuses are removed
         statuses = user.status_set.all()
-        self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer))
+        self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer.id))
 
     def get_activity_stream(self, user):
         """load the statuses to be displayed"""
         # clear unreads for this feed
-        r.set(self.unread_id(user), 0)
-        r.delete(self.unread_by_status_type_id(user))
+        r.set(self.unread_id(user.id), 0)
+        r.delete(self.unread_by_status_type_id(user.id))
 
-        statuses = self.get_store(self.stream_id(user))
+        statuses = self.get_store(self.stream_id(user.id))
         return (
             models.Status.objects.select_subclasses()
             .filter(id__in=statuses)
@@ -83,11 +91,11 @@ class ActivityStream(RedisStore):
 
     def get_unread_count(self, user):
         """get the unread status count for this user's feed"""
-        return int(r.get(self.unread_id(user)) or 0)
+        return int(r.get(self.unread_id(user.id)) or 0)
 
     def get_unread_count_by_status_type(self, user):
         """get the unread status count for this user's feed's status types"""
-        status_types = r.hgetall(self.unread_by_status_type_id(user))
+        status_types = r.hgetall(self.unread_by_status_type_id(user.id))
         return {
             str(key.decode("utf-8")): int(value) or 0
             for key, value in status_types.items()
@@ -95,13 +103,20 @@ class ActivityStream(RedisStore):
 
     def populate_streams(self, user):
         """go from zero to a timeline"""
-        self.populate_store(self.stream_id(user))
+        self.populate_store(self.stream_id(user.id))
 
-    def get_audience(self, status):  # pylint: disable=no-self-use
-        """given a status, what users should see it"""
-        # direct messages don't appeard in feeds, direct comments/reviews/etc do
+    @tracer.start_as_current_span("ActivityStream._get_audience")
+    def _get_audience(self, status):  # pylint: disable=no-self-use
+        """given a status, what users should see it, excluding the author"""
+        trace.get_current_span().set_attribute("status_type", status.status_type)
+        trace.get_current_span().set_attribute("status_privacy", status.privacy)
+        trace.get_current_span().set_attribute(
+            "status_reply_parent_privacy",
+            status.reply_parent.privacy if status.reply_parent else status.privacy,
+        )
+        # direct messages don't appear in feeds, direct comments/reviews/etc do
         if status.privacy == "direct" and status.status_type == "Note":
-            return []
+            return models.User.objects.none()
 
         # everybody who could plausibly see this status
         audience = models.User.objects.filter(
@@ -114,19 +129,38 @@ class ActivityStream(RedisStore):
         # only visible to the poster and mentioned users
         if status.privacy == "direct":
             audience = audience.filter(
-                Q(id=status.user.id)  # if the user is the post's author
-                | Q(id__in=status.mention_users.all())  # if the user is mentioned
+                Q(id__in=status.mention_users.all())  # if the user is mentioned
             )
+
+        # don't show replies to statuses the user can't see
+        elif status.reply_parent and status.reply_parent.privacy == "followers":
+            audience = audience.filter(
+                Q(id=status.reply_parent.user.id)  # if the user is the OG author
+                | (
+                    Q(following=status.user) & Q(following=status.reply_parent.user)
+                )  # if the user is following both authors
+            )
+
         # only visible to the poster's followers and tagged users
         elif status.privacy == "followers":
             audience = audience.filter(
-                Q(id=status.user.id)  # if the user is the post's author
-                | Q(following=status.user)  # if the user is following the author
+                Q(following=status.user)  # if the user is following the author
             )
-        return audience.distinct()
+        return audience.distinct("id")
 
-    def get_stores_for_object(self, obj):
-        return [self.stream_id(u) for u in self.get_audience(obj)]
+    @tracer.start_as_current_span("ActivityStream.get_audience")
+    def get_audience(self, status):
+        """given a status, what users should see it"""
+        trace.get_current_span().set_attribute("stream_id", self.key)
+        audience = self._get_audience(status).values_list("id", flat=True)
+        status_author = models.User.objects.filter(
+            is_active=True, local=True, id=status.user.id
+        ).values_list("id", flat=True)
+        return list(set(audience) | set(status_author))
+
+    def get_stores_for_users(self, user_ids):
+        """convert a list of user ids into redis store ids"""
+        return [self.stream_id(user_id) for user_id in user_ids]
 
     def get_statuses_for_user(self, user):  # pylint: disable=no-self-use
         """given a user, what statuses should they see on this stream"""
@@ -145,14 +179,17 @@ class HomeStream(ActivityStream):
 
     key = "home"
 
+    @tracer.start_as_current_span("HomeStream.get_audience")
     def get_audience(self, status):
-        audience = super().get_audience(status)
-        if not audience:
-            return []
-        return audience.filter(
-            Q(id=status.user.id)  # if the user is the post's author
-            | Q(following=status.user)  # if the user is following the author
-        ).distinct()
+        trace.get_current_span().set_attribute("stream_id", self.key)
+        audience = super()._get_audience(status)
+        # if the user is following the author
+        audience = audience.filter(following=status.user).values_list("id", flat=True)
+        # if the user is the post's author
+        status_author = models.User.objects.filter(
+            is_active=True, local=True, id=status.user.id
+        ).values_list("id", flat=True)
+        return list(set(audience) | set(status_author))
 
     def get_statuses_for_user(self, user):
         return models.Status.privacy_filter(
@@ -191,8 +228,18 @@ class BooksStream(ActivityStream):
 
     key = "books"
 
-    def get_audience(self, status):
+    def _get_audience(self, status):
         """anyone with the mentioned book on their shelves"""
+        work = (
+            status.book.parent_work
+            if hasattr(status, "book")
+            else status.mention_books.first().parent_work
+        )
+
+        audience = super()._get_audience(status)
+        return audience.filter(shelfbook__book__parent_work=work)
+
+    def get_audience(self, status):
         # only show public statuses on the books feed,
         # and only statuses that mention books
         if status.privacy != "public" or not (
@@ -200,16 +247,7 @@ class BooksStream(ActivityStream):
         ):
             return []
 
-        work = (
-            status.book.parent_work
-            if hasattr(status, "book")
-            else status.mention_books.first().parent_work
-        )
-
-        audience = super().get_audience(status)
-        if not audience:
-            return []
-        return audience.filter(shelfbook__book__parent_work=work).distinct()
+        return super().get_audience(status)
 
     def get_statuses_for_user(self, user):
         """any public status that mentions the user's books"""
@@ -233,38 +271,38 @@ class BooksStream(ActivityStream):
     def add_book_statuses(self, user, book):
         """add statuses about a book to a user's feed"""
         work = book.parent_work
-        statuses = (
-            models.Status.privacy_filter(
-                user,
-                privacy_levels=["public"],
-            )
-            .filter(
-                Q(comment__book__parent_work=work)
-                | Q(quotation__book__parent_work=work)
-                | Q(review__book__parent_work=work)
-                | Q(mention_books__parent_work=work)
-            )
-            .distinct()
+        statuses = models.Status.privacy_filter(
+            user,
+            privacy_levels=["public"],
         )
-        self.bulk_add_objects_to_store(statuses, self.stream_id(user))
+
+        book_comments = statuses.filter(Q(comment__book__parent_work=work))
+        book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
+        book_reviews = statuses.filter(Q(review__book__parent_work=work))
+        book_mentions = statuses.filter(Q(mention_books__parent_work=work))
+
+        self.bulk_add_objects_to_store(book_comments, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_quotations, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_reviews, self.stream_id(user.id))
+        self.bulk_add_objects_to_store(book_mentions, self.stream_id(user.id))
 
     def remove_book_statuses(self, user, book):
         """add statuses about a book to a user's feed"""
         work = book.parent_work
-        statuses = (
-            models.Status.privacy_filter(
-                user,
-                privacy_levels=["public"],
-            )
-            .filter(
-                Q(comment__book__parent_work=work)
-                | Q(quotation__book__parent_work=work)
-                | Q(review__book__parent_work=work)
-                | Q(mention_books__parent_work=work)
-            )
-            .distinct()
+        statuses = models.Status.privacy_filter(
+            user,
+            privacy_levels=["public"],
         )
-        self.bulk_remove_objects_from_store(statuses, self.stream_id(user))
+
+        book_comments = statuses.filter(Q(comment__book__parent_work=work))
+        book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
+        book_reviews = statuses.filter(Q(review__book__parent_work=work))
+        book_mentions = statuses.filter(Q(mention_books__parent_work=work))
+
+        self.bulk_remove_objects_from_store(book_comments, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_quotations, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_reviews, self.stream_id(user.id))
+        self.bulk_remove_objects_from_store(book_mentions, self.stream_id(user.id))
 
 
 # determine which streams are enabled in settings.py
@@ -287,6 +325,11 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
         remove_status_task.delay(instance.id)
         return
 
+    # We don't want to create multiple add_status_tasks for each status, and because
+    # the transactions are atomic, on_commit won't run until the status is ready to add.
+    if not created:
+        return
+
     # when creating new things, gotta wait on the transaction
     transaction.on_commit(
         lambda: add_status_on_create_command(sender, instance, created)
@@ -295,13 +338,21 @@ def add_status_on_create(sender, instance, created, *args, **kwargs):
 
 def add_status_on_create_command(sender, instance, created):
     """runs this code only after the database commit completes"""
-    priority = HIGH
+    # boosts trigger 'saves" twice, so don't bother duplicating the task
+    if sender == models.Boost and not created:
+        return
+
+    priority = STREAMS
     # check if this is an old status, de-prioritize if so
     # (this will happen if federation is very slow, or, more expectedly, on csv import)
     if instance.published_date < timezone.now() - timedelta(
         days=1
     ) or instance.created_date < instance.published_date - timedelta(days=1):
-        priority = LOW
+        # a backdated status from a local user is an import, don't add it
+        if instance.user.local:
+            return
+        # an out of date remote status is a low priority but should be added
+        priority = IMPORT_TRIGGERED
 
     add_status_task.apply_async(
         args=(instance.id,),
@@ -445,7 +496,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs):
 # ---- TASKS
 
 
-@app.task(queue=LOW)
+@app.task(queue=STREAMS)
 def add_book_statuses_task(user_id, book_id):
     """add statuses related to a book on shelve"""
     user = models.User.objects.get(id=user_id)
@@ -453,7 +504,7 @@ def add_book_statuses_task(user_id, book_id):
     BooksStream().add_book_statuses(user, book)
 
 
-@app.task(queue=LOW)
+@app.task(queue=STREAMS)
 def remove_book_statuses_task(user_id, book_id):
     """remove statuses about a book from a user's books feed"""
     user = models.User.objects.get(id=user_id)
@@ -461,7 +512,7 @@ def remove_book_statuses_task(user_id, book_id):
     BooksStream().remove_book_statuses(user, book)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def populate_stream_task(stream, user_id):
     """background task for populating an empty activitystream"""
     user = models.User.objects.get(id=user_id)
@@ -469,7 +520,7 @@ def populate_stream_task(stream, user_id):
     stream.populate_streams(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def remove_status_task(status_ids):
     """remove a status from any stream it might be in"""
     # this can take an id or a list of ids
@@ -479,10 +530,12 @@ def remove_status_task(status_ids):
 
     for stream in streams.values():
         for status in statuses:
-            stream.remove_object_from_related_stores(status)
+            stream.remove_object_from_stores(
+                status, stream.get_stores_for_users(stream.get_audience(status))
+            )
 
 
-@app.task(queue=HIGH)
+@app.task(queue=STREAMS)
 def add_status_task(status_id, increment_unread=False):
     """add a status to any stream it should be in"""
     status = models.Status.objects.select_subclasses().get(id=status_id)
@@ -494,7 +547,7 @@ def add_status_task(status_id, increment_unread=False):
         stream.add_status(status, increment_unread=increment_unread)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
     """remove all statuses by a user from a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -504,7 +557,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.remove_user_statuses(viewer, user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def add_user_statuses_task(viewer_id, user_id, stream_list=None):
     """add all statuses by a user to a viewer's stream"""
     stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@@ -514,7 +567,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None):
         stream.add_user_statuses(viewer, user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=STREAMS)
 def handle_boost_task(boost_id):
     """remove the original post and other, earlier boosts"""
     instance = models.Status.objects.get(id=boost_id)
@@ -528,10 +581,10 @@ def handle_boost_task(boost_id):
 
     for stream in streams.values():
         # people who should see the boost (not people who see the original status)
-        audience = stream.get_stores_for_object(instance)
-        stream.remove_object_from_related_stores(boosted, stores=audience)
+        audience = stream.get_stores_for_users(stream.get_audience(instance))
+        stream.remove_object_from_stores(boosted, audience)
         for status in old_versions:
-            stream.remove_object_from_related_stores(status, stores=audience)
+            stream.remove_object_from_stores(status, audience)
 
 
 def get_status_type(status):

bookwyrm/apps.py

@@ -1,4 +1,5 @@
 """Do further startup configuration and initialization"""
+
 import os
 import urllib
 import logging
@@ -14,16 +15,16 @@ def download_file(url, destination):
     """Downloads a file to the given path"""
     try:
         # Ensure our destination directory exists
-        os.makedirs(os.path.dirname(destination))
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
         with urllib.request.urlopen(url) as stream:
             with open(destination, "b+w") as outfile:
                 outfile.write(stream.read())
-    except (urllib.error.HTTPError, urllib.error.URLError):
-        logger.info("Failed to download file %s", url)
-    except OSError:
-        logger.info("Couldn't open font file %s for writing", destination)
-    except:  # pylint: disable=bare-except
-        logger.info("Unknown error in file download")
+    except (urllib.error.HTTPError, urllib.error.URLError) as err:
+        logger.error("Failed to download file %s: %s", url, err)
+    except OSError as err:
+        logger.error("Couldn't open font file %s for writing: %s", destination, err)
+    except Exception as err:  # pylint:disable=broad-except
+        logger.error("Unknown error in file download: %s", err)
 
 
 class BookwyrmConfig(AppConfig):
@@ -35,11 +36,12 @@ class BookwyrmConfig(AppConfig):
     # pylint: disable=no-self-use
     def ready(self):
         """set up OTLP and preview image files, if desired"""
-        if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
+        if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE:
             # pylint: disable=import-outside-toplevel
             from bookwyrm.telemetry import open_telemetry
 
             open_telemetry.instrumentDjango()
+            open_telemetry.instrumentPostgres()
 
         if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS:
             # Download any fonts that we don't have yet

bookwyrm/book_search.py

@@ -1,27 +1,69 @@
 """ using a bookwyrm instance as a source of book data """
+from __future__ import annotations
 from dataclasses import asdict, dataclass
 from functools import reduce
 import operator
+from typing import Optional, Union, Any, Literal, overload
 
 from django.contrib.postgres.search import SearchRank, SearchQuery
-from django.db.models import OuterRef, Subquery, F, Q
+from django.db.models import F, Q
+from django.db.models.query import QuerySet
 
 from bookwyrm import models
+from bookwyrm import connectors
 from bookwyrm.settings import MEDIA_FULL_URL
 
 
+@overload
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: Literal[False],
+) -> QuerySet[models.Edition]:
+    ...
+
+
+@overload
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: Literal[True],
+) -> Optional[models.Edition]:
+    ...
+
+
 # pylint: disable=arguments-differ
-def search(query, min_confidence=0, filters=None, return_first=False):
+def search(
+    query: str,
+    *,
+    min_confidence: float = 0,
+    filters: Optional[list[Any]] = None,
+    return_first: bool = False,
+    books: Optional[QuerySet[models.Edition]] = None,
+) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
     """search your local database"""
     filters = filters or []
     if not query:
-        return []
-    # first, try searching unqiue identifiers
-    results = search_identifiers(query, *filters, return_first=return_first)
+        return None if return_first else []
+    query = query.strip()
+
+    results = None
+    # first, try searching unique identifiers
+    # unique identifiers never have spaces, title/author usually do
+    if not " " in query:
+        results = search_identifiers(
+            query, *filters, return_first=return_first, books=books
+        )
+
+    # if there were no identifier results...
     if not results:
         # then try searching title/author
         results = search_title_author(
-            query, min_confidence, *filters, return_first=return_first
+            query, min_confidence, *filters, return_first=return_first, books=books
         )
     return results
 
@@ -30,26 +72,14 @@ def isbn_search(query):
     """search your local database"""
     if not query:
         return []
-
+    # Up-case the ISBN string to ensure any 'X' check-digit is correct
+    # If the ISBN has only 9 characters, prepend missing zero
+    query = query.strip().upper().rjust(10, "0")
     filters = [{f: query} for f in ["isbn_10", "isbn_13"]]
-    results = models.Edition.objects.filter(
+    return models.Edition.objects.filter(
         reduce(operator.or_, (Q(**f) for f in filters))
     ).distinct()
 
-    # when there are multiple editions of the same work, pick the default.
-    # it would be odd for this to happen.
-
-    default_editions = models.Edition.objects.filter(
-        parent_work=OuterRef("parent_work")
-    ).order_by("-edition_rank")
-    results = (
-        results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
-            default_id=F("id")
-        )
-        or results
-    )
-    return results
-
 
 def format_search_result(search_result):
     """convert a book object into a search result object"""
@@ -70,62 +100,65 @@ def format_search_result(search_result):
     ).json()
 
 
-def search_identifiers(query, *filters, return_first=False):
-    """tries remote_id, isbn; defined as dedupe fields on the model"""
+def search_identifiers(
+    query,
+    *filters,
+    return_first=False,
+    books=None,
+) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
+    """search Editions by deduplication fields
+
+    Best for cases when we can assume someone is searching for an exact match on
+    commonly unique data identifiers like isbn or specific library ids.
+    """
+    books = books or models.Edition.objects
+    if connectors.maybe_isbn(query):
+        # Oh did you think the 'S' in ISBN stood for 'standard'?
+        normalized_isbn = query.strip().upper().rjust(10, "0")
+        query = normalized_isbn
     # pylint: disable=W0212
     or_filters = [
         {f.name: query}
         for f in models.Edition._meta.get_fields()
         if hasattr(f, "deduplication_field") and f.deduplication_field
     ]
-    results = models.Edition.objects.filter(
+    results = books.filter(
         *filters, reduce(operator.or_, (Q(**f) for f in or_filters))
     ).distinct()
-    if results.count() <= 1:
-        if return_first:
-            return results.first()
-        return results
 
-    # when there are multiple editions of the same work, pick the default.
-    # it would be odd for this to happen.
-    default_editions = models.Edition.objects.filter(
-        parent_work=OuterRef("parent_work")
-    ).order_by("-edition_rank")
-    results = (
-        results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
-            default_id=F("id")
-        )
-        or results
-    )
     if return_first:
         return results.first()
     return results
 
 
-def search_title_author(query, min_confidence, *filters, return_first=False):
+def search_title_author(
+    query,
+    min_confidence,
+    *filters,
+    return_first=False,
+    books=None,
+) -> QuerySet[models.Edition]:
     """searches for title and author"""
+    books = books or models.Edition.objects
     query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
     results = (
-        models.Edition.objects.filter(*filters, search_vector=query)
+        books.filter(*filters, search_vector=query)
         .annotate(rank=SearchRank(F("search_vector"), query))
         .filter(rank__gt=min_confidence)
         .order_by("-rank")
     )
 
     # when there are multiple editions of the same work, pick the closest
-    editions_of_work = results.values("parent_work__id").values_list("parent_work__id")
+    editions_of_work = results.values_list("parent_work__id", flat=True).distinct()
 
     # filter out multiple editions of the same work
     list_results = []
-    for work_id in set(editions_of_work):
-        editions = results.filter(parent_work=work_id)
-        default = editions.order_by("-edition_rank").first()
-        default_rank = default.rank if default else 0
-        # if mutliple books have the top rank, pick the default edition
-        if default_rank == editions.first().rank:
-            result = default
-        else:
-            result = editions.first()
+    for work_id in editions_of_work[:30]:
+        result = (
+            results.filter(parent_work=work_id)
+            .order_by("-rank", "-edition_rank")
+            .first()
+        )
 
         if return_first:
             return result
@@ -140,11 +173,11 @@ class SearchResult:
     title: str
     key: str
     connector: object
-    view_link: str = None
-    author: str = None
-    year: str = None
-    cover: str = None
-    confidence: int = 1
+    view_link: Optional[str] = None
+    author: Optional[str] = None
+    year: Optional[str] = None
+    cover: Optional[str] = None
+    confidence: float = 1.0
 
     def __repr__(self):
         # pylint: disable=consider-using-f-string

bookwyrm/connectors/__init__.py

@@ -1,6 +1,6 @@
 """ bring connectors into the namespace """
 from .settings import CONNECTORS
 from .abstract_connector import ConnectorException
-from .abstract_connector import get_data, get_image
+from .abstract_connector import get_data, get_image, maybe_isbn
 
 from .connector_manager import search, first_search_result

bookwyrm/connectors/abstract_connector.py

@@ -1,89 +1,153 @@
 """ functionality outline for a book data connector """
+from __future__ import annotations
 from abc import ABC, abstractmethod
-import imghdr
+from typing import Optional, TypedDict, Any, Callable, Union, Iterator
+from urllib.parse import quote_plus
+
+# pylint: disable-next=deprecated-module
+import imghdr  # Deprecated in 3.11 for removal in 3.13; no good alternative yet
 import logging
 import re
+import asyncio
+import requests
+from requests.exceptions import RequestException
+import aiohttp
 
 from django.core.files.base import ContentFile
 from django.db import transaction
-import requests
-from requests.exceptions import RequestException
 
 from bookwyrm import activitypub, models, settings
+from bookwyrm.settings import USER_AGENT
 from .connector_manager import load_more_data, ConnectorException, raise_not_valid_url
 from .format_mappings import format_mappings
-
+from ..book_search import SearchResult
 
 logger = logging.getLogger(__name__)
 
+JsonDict = dict[str, Any]
+
+
+class ConnectorResults(TypedDict):
+    """TypedDict for results returned by connector"""
+
+    connector: AbstractMinimalConnector
+    results: list[SearchResult]
+
 
 class AbstractMinimalConnector(ABC):
     """just the bare bones, for other bookwyrm instances"""
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         # load connector settings
         info = models.Connector.objects.get(identifier=identifier)
         self.connector = info
 
         # the things in the connector model to copy over
-        self_fields = [
-            "base_url",
-            "books_url",
-            "covers_url",
-            "search_url",
-            "isbn_search_url",
-            "name",
-            "identifier",
-        ]
-        for field in self_fields:
-            setattr(self, field, getattr(info, field))
+        self.base_url = info.base_url
+        self.books_url = info.books_url
+        self.covers_url = info.covers_url
+        self.search_url = info.search_url
+        self.isbn_search_url = info.isbn_search_url
+        self.name = info.name
+        self.identifier = info.identifier
 
-    def get_search_url(self, query):
+    def get_search_url(self, query: str) -> str:
         """format the query url"""
         # Check if the query resembles an ISBN
         if maybe_isbn(query) and self.isbn_search_url and self.isbn_search_url != "":
-            return f"{self.isbn_search_url}{query}"
-
+            # Up-case the ISBN string to ensure any 'X' check-digit is correct
+            # If the ISBN has only 9 characters, prepend missing zero
+            normalized_query = query.strip().upper().rjust(10, "0")
+            return f"{self.isbn_search_url}{normalized_query}"
         # NOTE: previously, we tried searching isbn and if that produces no results,
         # searched as free text. This, instead, only searches isbn if it's isbn-y
-        return f"{self.search_url}{query}"
+        return f"{self.search_url}{quote_plus(query)}"
 
-    def process_search_response(self, query, data, min_confidence):
-        """Format the search results based on the formt of the query"""
+    def process_search_response(
+        self, query: str, data: Any, min_confidence: float
+    ) -> list[SearchResult]:
+        """Format the search results based on the format of the query"""
         if maybe_isbn(query):
             return list(self.parse_isbn_search_data(data))[:10]
         return list(self.parse_search_data(data, min_confidence))[:10]
 
+    async def get_results(
+        self,
+        session: aiohttp.ClientSession,
+        url: str,
+        min_confidence: float,
+        query: str,
+    ) -> Optional[ConnectorResults]:
+        """try this specific connector"""
+        # pylint: disable=line-too-long
+        headers = {
+            "Accept": (
+                'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
+            ),
+            "User-Agent": USER_AGENT,
+        }
+        params = {"min_confidence": min_confidence}
+        try:
+            async with session.get(url, headers=headers, params=params) as response:
+                if not response.ok:
+                    logger.info("Unable to connect to %s: %s", url, response.reason)
+                    return None
+
+                try:
+                    raw_data = await response.json()
+                except aiohttp.client_exceptions.ContentTypeError as err:
+                    logger.exception(err)
+                    return None
+
+                return ConnectorResults(
+                    connector=self,
+                    results=self.process_search_response(
+                        query, raw_data, min_confidence
+                    ),
+                )
+        except asyncio.TimeoutError:
+            logger.info("Connection timed out for url: %s", url)
+        except aiohttp.ClientError as err:
+            logger.info(err)
+        return None
+
     @abstractmethod
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
         """pull up a book record by whatever means possible"""
 
     @abstractmethod
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: Any, min_confidence: float
+    ) -> Iterator[SearchResult]:
         """turn the result json from a search into a list"""
 
     @abstractmethod
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(self, data: Any) -> Iterator[SearchResult]:
         """turn the result json from a search into a list"""
 
 
 class AbstractConnector(AbstractMinimalConnector):
     """generic book data connector"""
 
-    def __init__(self, identifier):
+    generated_remote_link_field = ""
+
+    def __init__(self, identifier: str):
         super().__init__(identifier)
         # fields we want to look for in book data to copy over
         # title we handle separately.
-        self.book_mappings = []
+        self.book_mappings: list[Mapping] = []
+        self.author_mappings: list[Mapping] = []
 
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> Optional[models.Book]:
         """translate arbitrary json into an Activitypub dataclass"""
         # first, check if we have the origin_id saved
         existing = models.Edition.find_existing_by_remote_id(
             remote_id
         ) or models.Work.find_existing_by_remote_id(remote_id)
         if existing:
-            if hasattr(existing, "default_edition"):
+            if hasattr(existing, "default_edition") and isinstance(
+                existing.default_edition, models.Edition
+            ):
                 return existing.default_edition
             return existing
 
@@ -115,6 +179,9 @@ class AbstractConnector(AbstractMinimalConnector):
             )
             # this will dedupe automatically
             work = work_activity.to_model(model=models.Work, overwrite=False)
+            if not work:
+                return None
+
             for author in self.get_authors_from_data(work_data):
                 work.authors.add(author)
 
@@ -122,12 +189,21 @@ class AbstractConnector(AbstractMinimalConnector):
         load_more_data.delay(self.connector.id, work.id)
         return edition
 
-    def get_book_data(self, remote_id):  # pylint: disable=no-self-use
+    def get_book_data(self, remote_id: str) -> JsonDict:  # pylint: disable=no-self-use
         """this allows connectors to override the default behavior"""
         return get_data(remote_id)
 
-    def create_edition_from_data(self, work, edition_data, instance=None):
+    def create_edition_from_data(
+        self,
+        work: models.Work,
+        edition_data: Union[str, JsonDict],
+        instance: Optional[models.Edition] = None,
+    ) -> Optional[models.Edition]:
         """if we already have the work, we're ready"""
+        if isinstance(edition_data, str):
+            # We don't expect a string here
+            return None
+
         mapped_data = dict_from_mappings(edition_data, self.book_mappings)
         mapped_data["work"] = work.remote_id
         edition_activity = activitypub.Edition(**mapped_data)
@@ -135,6 +211,9 @@ class AbstractConnector(AbstractMinimalConnector):
             model=models.Edition, overwrite=False, instance=instance
         )
 
+        if not edition:
+            return None
+
         # if we're updating an existing instance, we don't need to load authors
         if instance:
             return edition
@@ -151,7 +230,9 @@ class AbstractConnector(AbstractMinimalConnector):
 
         return edition
 
-    def get_or_create_author(self, remote_id, instance=None):
+    def get_or_create_author(
+        self, remote_id: str, instance: Optional[models.Author] = None
+    ) -> Optional[models.Author]:
         """load that author"""
         if not instance:
             existing = models.Author.find_existing_by_remote_id(remote_id)
@@ -171,46 +252,51 @@ class AbstractConnector(AbstractMinimalConnector):
             model=models.Author, overwrite=False, instance=instance
         )
 
-    def get_remote_id_from_model(self, obj):
+    def get_remote_id_from_model(self, obj: models.BookDataModel) -> Optional[str]:
         """given the data stored, how can we look this up"""
-        return getattr(obj, getattr(self, "generated_remote_link_field"))
+        remote_id: Optional[str] = getattr(obj, self.generated_remote_link_field)
+        return remote_id
 
-    def update_author_from_remote(self, obj):
+    def update_author_from_remote(self, obj: models.Author) -> Optional[models.Author]:
         """load the remote data from this connector and add it to an existing author"""
         remote_id = self.get_remote_id_from_model(obj)
+        if not remote_id:
+            return None
         return self.get_or_create_author(remote_id, instance=obj)
 
-    def update_book_from_remote(self, obj):
+    def update_book_from_remote(self, obj: models.Edition) -> Optional[models.Edition]:
         """load the remote data from this connector and add it to an existing book"""
         remote_id = self.get_remote_id_from_model(obj)
+        if not remote_id:
+            return None
         data = self.get_book_data(remote_id)
         return self.create_edition_from_data(obj.parent_work, data, instance=obj)
 
     @abstractmethod
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         """differentiate works and editions"""
 
     @abstractmethod
-    def get_edition_from_work_data(self, data):
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
         """every work needs at least one edition"""
 
     @abstractmethod
-    def get_work_from_edition_data(self, data):
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
         """every edition needs a work"""
 
     @abstractmethod
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         """load author data"""
 
     @abstractmethod
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         """get more info on a book"""
 
 
-def dict_from_mappings(data, mappings):
+def dict_from_mappings(data: JsonDict, mappings: list[Mapping]) -> JsonDict:
     """create a dict in Activitypub format, using mappings supplies by
     the subclass"""
-    result = {}
+    result: JsonDict = {}
     for mapping in mappings:
         # sometimes there are multiple mappings for one field, don't
         # overwrite earlier writes in that case
@@ -220,7 +306,11 @@ def dict_from_mappings(data, mappings):
     return result
 
 
-def get_data(url, params=None, timeout=10):
+def get_data(
+    url: str,
+    params: Optional[dict[str, str]] = None,
+    timeout: int = settings.QUERY_TIMEOUT,
+) -> JsonDict:
     """wrapper for request.get"""
     # check if the url is blocked
     raise_not_valid_url(url)
@@ -242,17 +332,26 @@ def get_data(url, params=None, timeout=10):
         raise ConnectorException(err)
 
     if not resp.ok:
-        raise ConnectorException()
+        if resp.status_code == 401:
+            # this is probably an AUTHORIZED_FETCH issue
+            resp.raise_for_status()
+        else:
+            raise ConnectorException()
     try:
         data = resp.json()
     except ValueError as err:
         logger.info(err)
         raise ConnectorException(err)
 
+    if not isinstance(data, dict):
+        raise ConnectorException("Unexpected data format")
+
     return data
 
 
-def get_image(url, timeout=10):
+def get_image(
+    url: str, timeout: int = 10
+) -> Union[tuple[ContentFile[bytes], str], tuple[None, None]]:
     """wrapper for requesting an image"""
     raise_not_valid_url(url)
     try:
@@ -282,14 +381,19 @@ def get_image(url, timeout=10):
 class Mapping:
     """associate a local database field with a field in an external dataset"""
 
-    def __init__(self, local_field, remote_field=None, formatter=None):
+    def __init__(
+        self,
+        local_field: str,
+        remote_field: Optional[str] = None,
+        formatter: Optional[Callable[[Any], Any]] = None,
+    ):
         noop = lambda x: x
 
         self.local_field = local_field
         self.remote_field = remote_field or local_field
         self.formatter = formatter or noop
 
-    def get_value(self, data):
+    def get_value(self, data: JsonDict) -> Optional[Any]:
         """pull a field from incoming json and return the formatted version"""
         value = data.get(self.remote_field)
         if not value:
@@ -300,7 +404,7 @@ class Mapping:
             return None
 
 
-def infer_physical_format(format_text):
+def infer_physical_format(format_text: str) -> Optional[str]:
     """try to figure out what the standardized format is from the free value"""
     format_text = format_text.lower()
     if format_text in format_mappings:
@@ -313,8 +417,8 @@ def infer_physical_format(format_text):
     return matches[0]
 
 
-def unique_physical_format(format_text):
-    """only store the format if it isn't diretly in the format mappings"""
+def unique_physical_format(format_text: str) -> Optional[str]:
+    """only store the format if it isn't directly in the format mappings"""
     format_text = format_text.lower()
     if format_text in format_mappings:
         # try a direct match, so saving this would be redundant
@@ -322,7 +426,14 @@ def unique_physical_format(format_text):
     return format_text
 
 
-def maybe_isbn(query):
+def maybe_isbn(query: str) -> bool:
     """check if a query looks like an isbn"""
     isbn = re.sub(r"[\W_]", "", query)  # removes filler characters
-    return len(isbn) in [10, 13]  # ISBN10 or ISBN13
+    # ISBNs must be numeric except an ISBN10 checkdigit can be 'X'
+    if not isbn.upper().rstrip("X").isnumeric():
+        return False
+    return len(isbn) in [
+        9,
+        10,
+        13,
+    ]  # ISBN10 or ISBN13, or maybe ISBN10 missing a leading zero

bookwyrm/connectors/bookwyrm_connector.py

@@ -1,4 +1,7 @@
 """ using another bookwyrm instance as a source of book data """
+from __future__ import annotations
+from typing import Any, Iterator
+
 from bookwyrm import activitypub, models
 from bookwyrm.book_search import SearchResult
 from .abstract_connector import AbstractMinimalConnector
@@ -7,15 +10,19 @@ from .abstract_connector import AbstractMinimalConnector
 class Connector(AbstractMinimalConnector):
     """this is basically just for search"""
 
-    def get_or_create_book(self, remote_id):
+    def get_or_create_book(self, remote_id: str) -> models.Edition:
         return activitypub.resolve_remote_id(remote_id, model=models.Edition)
 
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: list[dict[str, Any]], min_confidence: float
+    ) -> Iterator[SearchResult]:
         for search_result in data:
             search_result["connector"] = self
             yield SearchResult(**search_result)
 
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(
+        self, data: list[dict[str, Any]]
+    ) -> Iterator[SearchResult]:
         for search_result in data:
             search_result["connector"] = self
             yield SearchResult(**search_result)

bookwyrm/connectors/connector_manager.py

@@ -1,8 +1,11 @@
 """ interface with whatever connectors the app has """
+from __future__ import annotations
 import asyncio
 import importlib
 import ipaddress
 import logging
+from asyncio import Future
+from typing import Iterator, Any, Optional, Union, overload, Literal
 from urllib.parse import urlparse
 
 import aiohttp
@@ -12,8 +15,10 @@ from django.db.models import signals
 from requests import HTTPError
 
 from bookwyrm import book_search, models
-from bookwyrm.settings import SEARCH_TIMEOUT, USER_AGENT
-from bookwyrm.tasks import app
+from bookwyrm.book_search import SearchResult
+from bookwyrm.connectors import abstract_connector
+from bookwyrm.settings import SEARCH_TIMEOUT
+from bookwyrm.tasks import app, CONNECTORS
 
 logger = logging.getLogger(__name__)
 
@@ -22,61 +27,46 @@ class ConnectorException(HTTPError):
     """when the connector can't do what was asked"""
 
 
-async def get_results(session, url, min_confidence, query, connector):
-    """try this specific connector"""
-    # pylint: disable=line-too-long
-    headers = {
-        "Accept": (
-            'application/json, application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'
-        ),
-        "User-Agent": USER_AGENT,
-    }
-    params = {"min_confidence": min_confidence}
-    try:
-        async with session.get(url, headers=headers, params=params) as response:
-            if not response.ok:
-                logger.info("Unable to connect to %s: %s", url, response.reason)
-                return
-
-            try:
-                raw_data = await response.json()
-            except aiohttp.client_exceptions.ContentTypeError as err:
-                logger.exception(err)
-                return
-
-            return {
-                "connector": connector,
-                "results": connector.process_search_response(
-                    query, raw_data, min_confidence
-                ),
-            }
-    except asyncio.TimeoutError:
-        logger.info("Connection timed out for url: %s", url)
-    except aiohttp.ClientError as err:
-        logger.exception(err)
-
-
-async def async_connector_search(query, items, min_confidence):
+async def async_connector_search(
+    query: str,
+    items: list[tuple[str, abstract_connector.AbstractConnector]],
+    min_confidence: float,
+) -> list[Optional[abstract_connector.ConnectorResults]]:
     """Try a number of requests simultaneously"""
     timeout = aiohttp.ClientTimeout(total=SEARCH_TIMEOUT)
     async with aiohttp.ClientSession(timeout=timeout) as session:
-        tasks = []
+        tasks: list[Future[Optional[abstract_connector.ConnectorResults]]] = []
         for url, connector in items:
             tasks.append(
                 asyncio.ensure_future(
-                    get_results(session, url, min_confidence, query, connector)
+                    connector.get_results(session, url, min_confidence, query)
                 )
             )
 
         results = await asyncio.gather(*tasks)
-        return results
+        return list(results)
 
 
-def search(query, min_confidence=0.1, return_first=False):
-    """find books based on arbitary keywords"""
+@overload
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: Literal[False]
+) -> list[abstract_connector.ConnectorResults]:
+    ...
+
+
+@overload
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: Literal[True]
+) -> Optional[SearchResult]:
+    ...
+
+
+def search(
+    query: str, *, min_confidence: float = 0.1, return_first: bool = False
+) -> Union[list[abstract_connector.ConnectorResults], Optional[SearchResult]]:
+    """find books based on arbitrary keywords"""
     if not query:
-        return []
-    results = []
+        return None if return_first else []
 
     items = []
     for connector in get_connectors():
@@ -91,8 +81,12 @@ def search(query, min_confidence=0.1, return_first=False):
         items.append((url, connector))
 
     # load as many results as we can
-    results = asyncio.run(async_connector_search(query, items, min_confidence))
-    results = [r for r in results if r]
+    # failed requests will return None, so filter those out
+    results = [
+        r
+        for r in asyncio.run(async_connector_search(query, items, min_confidence))
+        if r
+    ]
 
     if return_first:
         # find the best result from all the responses and return that
@@ -100,11 +94,12 @@ def search(query, min_confidence=0.1, return_first=False):
         all_results = sorted(all_results, key=lambda r: r.confidence, reverse=True)
         return all_results[0] if all_results else None
 
-    # failed requests will return None, so filter those out
     return results
 
 
-def first_search_result(query, min_confidence=0.1):
+def first_search_result(
+    query: str, min_confidence: float = 0.1
+) -> Union[models.Edition, SearchResult, None]:
     """search until you find a result that fits"""
     # try local search first
     result = book_search.search(query, min_confidence=min_confidence, return_first=True)
@@ -114,18 +109,20 @@ def first_search_result(query, min_confidence=0.1):
     return search(query, min_confidence=min_confidence, return_first=True) or None
 
 
-def get_connectors():
+def get_connectors() -> Iterator[abstract_connector.AbstractConnector]:
     """load all connectors"""
     for info in models.Connector.objects.filter(active=True).order_by("priority").all():
         yield load_connector(info)
 
 
-def get_or_create_connector(remote_id):
+def get_or_create_connector(remote_id: str) -> abstract_connector.AbstractConnector:
     """get the connector related to the object's server"""
     url = urlparse(remote_id)
-    identifier = url.netloc
+    identifier = url.hostname
     if not identifier:
-        raise ValueError("Invalid remote id")
+        raise ValueError(f"Invalid remote id: {remote_id}")
+
+    base_url = f"{url.scheme}://{url.netloc}"
 
     try:
         connector_info = models.Connector.objects.get(identifier=identifier)
@@ -133,18 +130,18 @@ def get_or_create_connector(remote_id):
         connector_info = models.Connector.objects.create(
             identifier=identifier,
             connector_file="bookwyrm_connector",
-            base_url=f"https://{identifier}",
-            books_url=f"https://{identifier}/book",
-            covers_url=f"https://{identifier}/images/covers",
-            search_url=f"https://{identifier}/search?q=",
+            base_url=base_url,
+            books_url=f"{base_url}/book",
+            covers_url=f"{base_url}/images/covers",
+            search_url=f"{base_url}/search?q=",
             priority=2,
         )
 
     return load_connector(connector_info)
 
 
-@app.task(queue="low_priority")
-def load_more_data(connector_id, book_id):
+@app.task(queue=CONNECTORS)
+def load_more_data(connector_id: str, book_id: str) -> None:
     """background the work of getting all 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
     connector = load_connector(connector_info)
@@ -152,8 +149,10 @@ def load_more_data(connector_id, book_id):
     connector.expand_book_data(book)
 
 
-@app.task(queue="low_priority")
-def create_edition_task(connector_id, work_id, data):
+@app.task(queue=CONNECTORS)
+def create_edition_task(
+    connector_id: int, work_id: int, data: Union[str, abstract_connector.JsonDict]
+) -> None:
     """separate task for each of the 10,000 editions of LoTR"""
     connector_info = models.Connector.objects.get(id=connector_id)
     connector = load_connector(connector_info)
@@ -161,30 +160,41 @@ def create_edition_task(connector_id, work_id, data):
     connector.create_edition_from_data(work, data)
 
 
-def load_connector(connector_info):
+def load_connector(
+    connector_info: models.Connector,
+) -> abstract_connector.AbstractConnector:
     """instantiate the connector class"""
     connector = importlib.import_module(
         f"bookwyrm.connectors.{connector_info.connector_file}"
     )
-    return connector.Connector(connector_info.identifier)
+    return connector.Connector(connector_info.identifier)  # type: ignore[no-any-return]
 
 
 @receiver(signals.post_save, sender="bookwyrm.FederatedServer")
 # pylint: disable=unused-argument
-def create_connector(sender, instance, created, *args, **kwargs):
+def create_connector(
+    sender: Any,
+    instance: models.FederatedServer,
+    created: Any,
+    *args: Any,
+    **kwargs: Any,
+) -> None:
     """create a connector to an external bookwyrm server"""
     if instance.application_type == "bookwyrm":
         get_or_create_connector(f"https://{instance.server_name}")
 
 
-def raise_not_valid_url(url):
+def raise_not_valid_url(url: str) -> None:
     """do some basic reality checks on the url"""
     parsed = urlparse(url)
     if not parsed.scheme in ["http", "https"]:
         raise ConnectorException("Invalid scheme: ", url)
 
+    if not parsed.hostname:
+        raise ConnectorException("Hostname missing: ", url)
+
     try:
-        ipaddress.ip_address(parsed.netloc)
+        ipaddress.ip_address(parsed.hostname)
         raise ConnectorException("Provided url is an IP address: ", url)
     except ValueError:
         # it's not an IP address, which is good

bookwyrm/connectors/inventaire.py

@@ -1,9 +1,10 @@
 """ inventaire data connector """
 import re
+from typing import Any, Union, Optional, Iterator, Iterable
 
 from bookwyrm import models
 from bookwyrm.book_search import SearchResult
-from .abstract_connector import AbstractConnector, Mapping
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
 from .abstract_connector import get_data
 from .connector_manager import ConnectorException, create_edition_task
 
@@ -13,7 +14,7 @@ class Connector(AbstractConnector):
 
     generated_remote_link_field = "inventaire_id"
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         super().__init__(identifier)
 
         get_first = lambda a: a[0]
@@ -60,13 +61,13 @@ class Connector(AbstractConnector):
             Mapping("died", remote_field="wdt:P570", formatter=get_first),
         ] + shared_mappings
 
-    def get_remote_id(self, value):
+    def get_remote_id(self, value: str) -> str:
         """convert an id/uri into a url"""
         return f"{self.books_url}?action=by-uris&uris={value}"
 
-    def get_book_data(self, remote_id):
+    def get_book_data(self, remote_id: str) -> JsonDict:
         data = get_data(remote_id)
-        extracted = list(data.get("entities").values())
+        extracted = list(data.get("entities", {}).values())
         try:
             data = extracted[0]
         except (KeyError, IndexError):
@@ -74,10 +75,16 @@ class Connector(AbstractConnector):
         # flatten the data so that images, uri, and claims are on the same level
         return {
             **data.get("claims", {}),
-            **{k: data.get(k) for k in ["uri", "image", "labels", "sitelinks", "type"]},
+            **{
+                k: data.get(k)
+                for k in ["uri", "image", "labels", "sitelinks", "type"]
+                if k in data
+            },
         }
 
-    def parse_search_data(self, data, min_confidence):
+    def parse_search_data(
+        self, data: JsonDict, min_confidence: float
+    ) -> Iterator[SearchResult]:
         for search_result in data.get("results", []):
             images = search_result.get("image")
             cover = f"{self.covers_url}/img/entities/{images[0]}" if images else None
@@ -96,8 +103,8 @@ class Connector(AbstractConnector):
                 connector=self,
             )
 
-    def parse_isbn_search_data(self, data):
-        """got some daaaata"""
+    def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
+        """got some data"""
         results = data.get("entities")
         if not results:
             return
@@ -114,35 +121,44 @@ class Connector(AbstractConnector):
                 connector=self,
             )
 
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         return data.get("type") == "work"
 
-    def load_edition_data(self, work_uri):
+    def load_edition_data(self, work_uri: str) -> JsonDict:
         """get a list of editions for a work"""
         # pylint: disable=line-too-long
         url = f"{self.books_url}?action=reverse-claims&property=wdt:P629&value={work_uri}&sort=true"
         return get_data(url)
 
-    def get_edition_from_work_data(self, data):
-        data = self.load_edition_data(data.get("uri"))
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
+        work_uri = data.get("uri")
+        if not work_uri:
+            raise ConnectorException("Invalid URI")
+        data = self.load_edition_data(work_uri)
         try:
             uri = data.get("uris", [])[0]
         except IndexError:
             raise ConnectorException("Invalid book data")
         return self.get_book_data(self.get_remote_id(uri))
 
-    def get_work_from_edition_data(self, data):
-        uri = data.get("wdt:P629", [None])[0]
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
+        try:
+            uri = data.get("wdt:P629", [])[0]
+        except IndexError:
+            raise ConnectorException("Invalid book data")
+
         if not uri:
             raise ConnectorException("Invalid book data")
         return self.get_book_data(self.get_remote_id(uri))
 
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         authors = data.get("wdt:P50", [])
         for author in authors:
-            yield self.get_or_create_author(self.get_remote_id(author))
+            model = self.get_or_create_author(self.get_remote_id(author))
+            if model:
+                yield model
 
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         work = book
         # go from the edition to the work, if necessary
         if isinstance(book, models.Edition):
@@ -154,36 +170,45 @@ class Connector(AbstractConnector):
             # who knows, man
             return
 
-        for edition_uri in edition_options.get("uris"):
+        for edition_uri in edition_options.get("uris", []):
             remote_id = self.get_remote_id(edition_uri)
             create_edition_task.delay(self.connector.id, work.id, remote_id)
 
-    def create_edition_from_data(self, work, edition_data, instance=None):
+    def create_edition_from_data(
+        self,
+        work: models.Work,
+        edition_data: Union[str, JsonDict],
+        instance: Optional[models.Edition] = None,
+    ) -> Optional[models.Edition]:
         """pass in the url as data and then call the version in abstract connector"""
         if isinstance(edition_data, str):
             try:
                 edition_data = self.get_book_data(edition_data)
             except ConnectorException:
                 # who, indeed, knows
-                return
-        super().create_edition_from_data(work, edition_data, instance=instance)
+                return None
+        return super().create_edition_from_data(work, edition_data, instance=instance)
 
-    def get_cover_url(self, cover_blob, *_):
+    def get_cover_url(
+        self, cover_blob: Union[list[JsonDict], JsonDict], *_: Any
+    ) -> Optional[str]:
         """format the relative cover url into an absolute one:
         {"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"}
         """
         # covers may or may not be a list
-        if isinstance(cover_blob, list) and len(cover_blob) > 0:
+        if isinstance(cover_blob, list):
+            if len(cover_blob) == 0:
+                return None
             cover_blob = cover_blob[0]
         cover_id = cover_blob.get("url")
-        if not cover_id:
+        if not isinstance(cover_id, str):
             return None
         # cover may or may not be an absolute url already
         if re.match(r"^http", cover_id):
             return cover_id
         return f"{self.covers_url}{cover_id}"
 
-    def resolve_keys(self, keys):
+    def resolve_keys(self, keys: Iterable[str]) -> list[str]:
         """cool, it's "wd:Q3156592" now what the heck does that mean"""
         results = []
         for uri in keys:
@@ -191,10 +216,10 @@ class Connector(AbstractConnector):
                 data = self.get_book_data(self.get_remote_id(uri))
             except ConnectorException:
                 continue
-            results.append(get_language_code(data.get("labels")))
+            results.append(get_language_code(data.get("labels", {})))
         return results
 
-    def get_description(self, links):
+    def get_description(self, links: JsonDict) -> str:
         """grab an extracted excerpt from wikipedia"""
         link = links.get("enwiki")
         if not link:
@@ -204,15 +229,15 @@ class Connector(AbstractConnector):
             data = get_data(url)
         except ConnectorException:
             return ""
-        return data.get("extract")
+        return data.get("extract", "")
 
-    def get_remote_id_from_model(self, obj):
+    def get_remote_id_from_model(self, obj: models.BookDataModel) -> str:
         """use get_remote_id to figure out the link from a model obj"""
         remote_id_value = obj.inventaire_id
         return self.get_remote_id(remote_id_value)
 
 
-def get_language_code(options, code="en"):
+def get_language_code(options: JsonDict, code: str = "en") -> Any:
     """when there are a bunch of translation but we need a single field"""
     result = options.get(code)
     if result:

bookwyrm/connectors/openlibrary.py

@@ -1,9 +1,13 @@
 """ openlibrary data connector """
 import re
+from typing import Any, Optional, Union, Iterator, Iterable
+
+from markdown import markdown
 
 from bookwyrm import models
 from bookwyrm.book_search import SearchResult
-from .abstract_connector import AbstractConnector, Mapping
+from bookwyrm.utils.sanitizer import clean
+from .abstract_connector import AbstractConnector, Mapping, JsonDict
 from .abstract_connector import get_data, infer_physical_format, unique_physical_format
 from .connector_manager import ConnectorException, create_edition_task
 from .openlibrary_languages import languages
@@ -14,7 +18,7 @@ class Connector(AbstractConnector):
 
     generated_remote_link_field = "openlibrary_link"
 
-    def __init__(self, identifier):
+    def __init__(self, identifier: str):
         super().__init__(identifier)
 
         get_first = lambda a, *args: a[0]
@@ -94,14 +98,14 @@ class Connector(AbstractConnector):
             Mapping("inventaire_id", remote_field="links", formatter=get_inventaire_id),
         ]
 
-    def get_book_data(self, remote_id):
+    def get_book_data(self, remote_id: str) -> JsonDict:
         data = get_data(remote_id)
         if data.get("type", {}).get("key") == "/type/redirect":
-            remote_id = self.base_url + data.get("location")
+            remote_id = self.base_url + data.get("location", "")
             return get_data(remote_id)
         return data
 
-    def get_remote_id_from_data(self, data):
+    def get_remote_id_from_data(self, data: JsonDict) -> str:
         """format a url from an openlibrary id field"""
         try:
             key = data["key"]
@@ -109,10 +113,10 @@ class Connector(AbstractConnector):
             raise ConnectorException("Invalid book data")
         return f"{self.books_url}{key}"
 
-    def is_work_data(self, data):
+    def is_work_data(self, data: JsonDict) -> bool:
         return bool(re.match(r"^[\/\w]+OL\d+W$", data["key"]))
 
-    def get_edition_from_work_data(self, data):
+    def get_edition_from_work_data(self, data: JsonDict) -> JsonDict:
         try:
             key = data["key"]
         except KeyError:
@@ -124,7 +128,7 @@ class Connector(AbstractConnector):
             raise ConnectorException("No editions for work")
         return edition
 
-    def get_work_from_edition_data(self, data):
+    def get_work_from_edition_data(self, data: JsonDict) -> JsonDict:
         try:
             key = data["works"][0]["key"]
         except (IndexError, KeyError):
@@ -132,7 +136,7 @@ class Connector(AbstractConnector):
         url = f"{self.books_url}{key}"
         return self.get_book_data(url)
 
-    def get_authors_from_data(self, data):
+    def get_authors_from_data(self, data: JsonDict) -> Iterator[models.Author]:
         """parse author json and load or create authors"""
         for author_blob in data.get("authors", []):
             author_blob = author_blob.get("author", author_blob)
@@ -144,7 +148,7 @@ class Connector(AbstractConnector):
                 continue
             yield author
 
-    def get_cover_url(self, cover_blob, size="L"):
+    def get_cover_url(self, cover_blob: list[str], size: str = "L") -> Optional[str]:
         """ask openlibrary for the cover"""
         if not cover_blob:
             return None
@@ -152,8 +156,10 @@ class Connector(AbstractConnector):
         image_name = f"{cover_id}-{size}.jpg"
         return f"{self.covers_url}/b/id/{image_name}"
 
-    def parse_search_data(self, data, min_confidence):
-        for idx, search_result in enumerate(data.get("docs")):
+    def parse_search_data(
+        self, data: JsonDict, min_confidence: float
+    ) -> Iterator[SearchResult]:
+        for idx, search_result in enumerate(data.get("docs", [])):
             # build the remote id from the openlibrary key
             key = self.books_url + search_result["key"]
             author = search_result.get("author_name") or ["Unknown"]
@@ -174,7 +180,7 @@ class Connector(AbstractConnector):
                 confidence=confidence,
             )
 
-    def parse_isbn_search_data(self, data):
+    def parse_isbn_search_data(self, data: JsonDict) -> Iterator[SearchResult]:
         for search_result in list(data.values()):
             # build the remote id from the openlibrary key
             key = self.books_url + search_result["key"]
@@ -188,12 +194,12 @@ class Connector(AbstractConnector):
                 year=search_result.get("publish_date"),
             )
 
-    def load_edition_data(self, olkey):
+    def load_edition_data(self, olkey: str) -> JsonDict:
         """query openlibrary for editions of a work"""
         url = f"{self.books_url}/works/{olkey}/editions"
         return self.get_book_data(url)
 
-    def expand_book_data(self, book):
+    def expand_book_data(self, book: models.Book) -> None:
         work = book
         # go from the edition to the work, if necessary
         if isinstance(book, models.Edition):
@@ -206,14 +212,14 @@ class Connector(AbstractConnector):
             # who knows, man
             return
 
-        for edition_data in edition_options.get("entries"):
+        for edition_data in edition_options.get("entries", []):
             # does this edition have ANY interesting data?
             if ignore_edition(edition_data):
                 continue
             create_edition_task.delay(self.connector.id, work.id, edition_data)
 
 
-def ignore_edition(edition_data):
+def ignore_edition(edition_data: JsonDict) -> bool:
     """don't load a million editions that have no metadata"""
     # an isbn, we love to see it
     if edition_data.get("isbn_13") or edition_data.get("isbn_10"):
@@ -232,19 +238,30 @@ def ignore_edition(edition_data):
     return True
 
 
-def get_description(description_blob):
+def get_description(description_blob: Union[JsonDict, str]) -> str:
     """descriptions can be a string or a dict"""
     if isinstance(description_blob, dict):
-        return description_blob.get("value")
-    return description_blob
+        description = markdown(description_blob.get("value", ""))
+    else:
+        description = markdown(description_blob)
+
+    if (
+        description.startswith("<p>")
+        and description.endswith("</p>")
+        and description.count("<p>") == 1
+    ):
+        # If there is just one <p> tag and it is around the text remove it
+        return description[len("<p>") : -len("</p>")].strip()
+
+    return clean(description)
 
 
-def get_openlibrary_key(key):
+def get_openlibrary_key(key: str) -> str:
     """convert /books/OL27320736M into OL27320736M"""
     return key.split("/")[-1]
 
 
-def get_languages(language_blob):
+def get_languages(language_blob: Iterable[JsonDict]) -> list[Optional[str]]:
     """/language/eng -> English"""
     langs = []
     for lang in language_blob:
@@ -252,14 +269,14 @@ def get_languages(language_blob):
     return langs
 
 
-def get_dict_field(blob, field_name):
+def get_dict_field(blob: Optional[JsonDict], field_name: str) -> Optional[Any]:
     """extract the isni from the remote id data for the author"""
     if not blob or not isinstance(blob, dict):
         return None
     return blob.get(field_name)
 
 
-def get_wikipedia_link(links):
+def get_wikipedia_link(links: list[Any]) -> Optional[str]:
     """extract wikipedia links"""
     if not isinstance(links, list):
         return None
@@ -272,7 +289,7 @@ def get_wikipedia_link(links):
     return None
 
 
-def get_inventaire_id(links):
+def get_inventaire_id(links: list[Any]) -> Optional[str]:
     """extract and format inventaire ids"""
     if not isinstance(links, list):
         return None
@@ -282,11 +299,13 @@ def get_inventaire_id(links):
             continue
         if link.get("title") == "inventaire.io":
             iv_link = link.get("url")
+            if not isinstance(iv_link, str):
+                return None
             return iv_link.split("/")[-1]
     return None
 
 
-def pick_default_edition(options):
+def pick_default_edition(options: list[JsonDict]) -> Optional[JsonDict]:
     """favor physical copies with covers in english"""
     if not options:
         return None

bookwyrm/emailing.py

@@ -3,8 +3,8 @@ from django.core.mail import EmailMultiAlternatives
 from django.template.loader import get_template
 
 from bookwyrm import models, settings
-from bookwyrm.tasks import app
-from bookwyrm.settings import DOMAIN
+from bookwyrm.tasks import app, EMAIL
+from bookwyrm.settings import DOMAIN, BASE_URL
 
 
 def email_data():
@@ -14,16 +14,23 @@ def email_data():
         "site_name": site.name,
         "logo": site.logo_small_url,
         "domain": DOMAIN,
+        "base_url": BASE_URL,
         "user": None,
     }
 
 
+def test_email(user):
+    """Just an admin checking if emails are sending"""
+    data = email_data()
+    send_email(user.email, *format_email("test", data))
+
+
 def email_confirmation_email(user):
     """newly registered users confirm email address"""
     data = email_data()
     data["confirmation_code"] = user.confirmation_code
     data["confirmation_link"] = user.confirmation_link
-    send_email.delay(user.email, *format_email("confirm", data))
+    send_email(user.email, *format_email("confirm", data))
 
 
 def invite_email(invite_request):
@@ -38,7 +45,7 @@ def password_reset_email(reset_code):
     data = email_data()
     data["reset_link"] = reset_code.link
     data["user"] = reset_code.user.display_name
-    send_email.delay(reset_code.user.email, *format_email("password_reset", data))
+    send_email(reset_code.user.email, *format_email("password_reset", data))
 
 
 def moderation_report_email(report):
@@ -48,6 +55,7 @@ def moderation_report_email(report):
     if report.user:
         data["reportee"] = report.user.localname or report.user.username
     data["report_link"] = report.remote_id
+    data["link_domain"] = report.links.exists()
 
     for admin in models.User.objects.filter(
         groups__name__in=["admin", "moderator"]
@@ -68,7 +76,7 @@ def format_email(email_name, data):
     return (subject, html_content, text_content)
 
 
-@app.task(queue="high_priority")
+@app.task(queue=EMAIL)
 def send_email(recipient, subject, html_content, text_content):
     """use a task to send the email"""
     email = EmailMultiAlternatives(

bookwyrm/forms/admin.py

@@ -2,19 +2,20 @@
 import datetime
 
 from django import forms
+from django.core.exceptions import PermissionDenied
 from django.forms import widgets
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from django_celery_beat.models import IntervalSchedule
 
 from bookwyrm import models
-from .custom_form import CustomForm
+from .custom_form import CustomForm, StyledForm
 
 
 # pylint: disable=missing-class-docstring
 class ExpiryWidget(widgets.Select):
     def value_from_datadict(self, data, files, name):
-        """human-readable exiration time buckets"""
+        """human-readable expiration time buckets"""
         selected_string = super().value_from_datadict(data, files, name)
 
         if selected_string == "day":
@@ -54,11 +55,46 @@ class CreateInviteForm(CustomForm):
 class SiteForm(CustomForm):
     class Meta:
         model = models.SiteSettings
-        exclude = ["admin_code", "install_mode"]
+        fields = [
+            "name",
+            "instance_tagline",
+            "instance_description",
+            "instance_short_description",
+            "default_theme",
+            "code_of_conduct",
+            "privacy_policy",
+            "impressum",
+            "show_impressum",
+            "logo",
+            "logo_small",
+            "favicon",
+            "support_link",
+            "support_title",
+            "admin_email",
+            "footer_item",
+        ]
         widgets = {
             "instance_short_description": forms.TextInput(
                 attrs={"aria-describedby": "desc_instance_short_description"}
             ),
+        }
+
+
+class RegistrationForm(CustomForm):
+    class Meta:
+        model = models.SiteSettings
+        fields = [
+            "allow_registration",
+            "allow_invite_requests",
+            "registration_closed_text",
+            "invite_request_text",
+            "invite_request_question",
+            "invite_question_text",
+            "require_confirm_email",
+            "default_user_auth_group",
+        ]
+
+        widgets = {
             "require_confirm_email": forms.CheckboxInput(
                 attrs={"aria-describedby": "desc_require_confirm_email"}
             ),
@@ -68,6 +104,23 @@ class SiteForm(CustomForm):
         }
 
 
+class RegistrationLimitedForm(CustomForm):
+    class Meta:
+        model = models.SiteSettings
+        fields = [
+            "registration_closed_text",
+            "invite_request_text",
+            "invite_request_question",
+            "invite_question_text",
+        ]
+
+        widgets = {
+            "invite_request_text": forms.Textarea(
+                attrs={"aria-describedby": "desc_invite_request_text"}
+            ),
+        }
+
+
 class ThemeForm(CustomForm):
     class Meta:
         model = models.Theme
@@ -130,7 +183,7 @@ class AutoModRuleForm(CustomForm):
         fields = ["string_match", "flag_users", "flag_statuses", "created_by"]
 
 
-class IntervalScheduleForm(CustomForm):
+class IntervalScheduleForm(StyledForm):
     class Meta:
         model = IntervalSchedule
         fields = ["every", "period"]
@@ -139,3 +192,10 @@ class IntervalScheduleForm(CustomForm):
             "every": forms.NumberInput(attrs={"aria-describedby": "desc_every"}),
             "period": forms.Select(attrs={"aria-describedby": "desc_period"}),
         }
+
+    # pylint: disable=arguments-differ
+    def save(self, request, *args, **kwargs):
+        """This is an outside model so the perms check works differently"""
+        if not request.user.has_perm("bookwyrm.moderate_user"):
+            raise PermissionDenied()
+        return super().save(*args, **kwargs)

bookwyrm/forms/author.py

@@ -15,12 +15,15 @@ class AuthorForm(CustomForm):
             "aliases",
             "bio",
             "wikipedia_link",
+            "wikidata",
+            "website",
             "born",
             "died",
             "openlibrary_key",
             "inventaire_id",
             "librarything_key",
             "goodreads_key",
+            "isfdb",
             "isni",
         ]
         widgets = {
@@ -30,10 +33,12 @@ class AuthorForm(CustomForm):
             "wikipedia_link": forms.TextInput(
                 attrs={"aria-describedby": "desc_wikipedia_link"}
             ),
+            "wikidata": forms.TextInput(attrs={"aria-describedby": "desc_wikidata"}),
+            "website": forms.TextInput(attrs={"aria-describedby": "desc_website"}),
             "born": forms.SelectDateWidget(attrs={"aria-describedby": "desc_born"}),
             "died": forms.SelectDateWidget(attrs={"aria-describedby": "desc_died"}),
-            "oepnlibrary_key": forms.TextInput(
-                attrs={"aria-describedby": "desc_oepnlibrary_key"}
+            "openlibrary_key": forms.TextInput(
+                attrs={"aria-describedby": "desc_openlibrary_key"}
             ),
             "inventaire_id": forms.TextInput(
                 attrs={"aria-describedby": "desc_inventaire_id"}

bookwyrm/forms/books.py

@@ -1,8 +1,9 @@
 """ using django model forms """
 from django import forms
 
+from file_resubmit.widgets import ResubmitImageWidget
+
 from bookwyrm import models
-from bookwyrm.models.fields import ClearableFileInputWithWarning
 from .custom_form import CustomForm
 from .widgets import ArrayWidget, SelectDateWidget, Select
 
@@ -18,22 +19,37 @@ class CoverForm(CustomForm):
 class EditionForm(CustomForm):
     class Meta:
         model = models.Edition
-        exclude = [
-            "remote_id",
-            "origin_id",
-            "created_date",
-            "updated_date",
-            "edition_rank",
-            "authors",
-            "parent_work",
-            "shelves",
-            "connector",
-            "search_vector",
-            "links",
-            "file_links",
+        fields = [
+            "title",
+            "sort_title",
+            "subtitle",
+            "description",
+            "series",
+            "series_number",
+            "languages",
+            "subjects",
+            "publishers",
+            "first_published_date",
+            "published_date",
+            "cover",
+            "physical_format",
+            "physical_format_detail",
+            "pages",
+            "isbn_13",
+            "isbn_10",
+            "openlibrary_key",
+            "inventaire_id",
+            "goodreads_key",
+            "oclc_number",
+            "asin",
+            "aasin",
+            "isfdb",
         ]
         widgets = {
             "title": forms.TextInput(attrs={"aria-describedby": "desc_title"}),
+            "sort_title": forms.TextInput(
+                attrs={"aria-describedby": "desc_sort_title"}
+            ),
             "subtitle": forms.TextInput(attrs={"aria-describedby": "desc_subtitle"}),
             "description": forms.Textarea(
                 attrs={"aria-describedby": "desc_description"}
@@ -55,9 +71,7 @@ class EditionForm(CustomForm):
             "published_date": SelectDateWidget(
                 attrs={"aria-describedby": "desc_published_date"}
             ),
-            "cover": ClearableFileInputWithWarning(
-                attrs={"aria-describedby": "desc_cover"}
-            ),
+            "cover": ResubmitImageWidget(attrs={"aria-describedby": "desc_cover"}),
             "physical_format": Select(
                 attrs={"aria-describedby": "desc_physical_format"}
             ),
@@ -73,10 +87,15 @@ class EditionForm(CustomForm):
             "inventaire_id": forms.TextInput(
                 attrs={"aria-describedby": "desc_inventaire_id"}
             ),
+            "goodreads_key": forms.TextInput(
+                attrs={"aria-describedby": "desc_goodreads_key"}
+            ),
             "oclc_number": forms.TextInput(
                 attrs={"aria-describedby": "desc_oclc_number"}
             ),
             "ASIN": forms.TextInput(attrs={"aria-describedby": "desc_ASIN"}),
+            "AASIN": forms.TextInput(attrs={"aria-describedby": "desc_AASIN"}),
+            "isfdb": forms.TextInput(attrs={"aria-describedby": "desc_isfdb"}),
         }
 
 
@@ -91,6 +110,7 @@ class EditionFromWorkForm(CustomForm):
         model = models.Work
         fields = [
             "title",
+            "sort_title",
             "subtitle",
             "authors",
             "description",

bookwyrm/forms/custom_form.py

@@ -4,7 +4,7 @@ from django.forms import ModelForm
 from django.forms.widgets import Textarea
 
 
-class CustomForm(ModelForm):
+class StyledForm(ModelForm):
     """add css classes to the forms"""
 
     def __init__(self, *args, **kwargs):
@@ -16,7 +16,7 @@ class CustomForm(ModelForm):
         css_classes["checkbox"] = "checkbox"
         css_classes["textarea"] = "textarea"
         # pylint: disable=super-with-arguments
-        super(CustomForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         for visible in self.visible_fields():
             if hasattr(visible.field.widget, "input_type"):
                 input_type = visible.field.widget.input_type
@@ -24,3 +24,13 @@ class CustomForm(ModelForm):
                 input_type = "textarea"
                 visible.field.widget.attrs["rows"] = 5
             visible.field.widget.attrs["class"] = css_classes[input_type]
+
+
+class CustomForm(StyledForm):
+    """Check permissions on save"""
+
+    # pylint: disable=arguments-differ
+    def save(self, request, *args, **kwargs):
+        """Save and check perms"""
+        self.instance.raise_not_editable(request.user)
+        return super().save(*args, **kwargs)

bookwyrm/forms/edit_user.py

@@ -1,11 +1,13 @@
 """ using django model forms """
 from django import forms
+from django.contrib.auth.password_validation import validate_password
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext_lazy as _
 
 from bookwyrm import models
 from bookwyrm.models.fields import ClearableFileInputWithWarning
 from .custom_form import CustomForm
 
-
 # pylint: disable=missing-class-docstring
 class EditUserForm(CustomForm):
     class Meta:
@@ -66,3 +68,67 @@ class DeleteUserForm(CustomForm):
     class Meta:
         model = models.User
         fields = ["password"]
+
+
+class MoveUserForm(CustomForm):
+    target = forms.CharField(widget=forms.TextInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+
+
+class AliasUserForm(CustomForm):
+    username = forms.CharField(widget=forms.TextInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+
+
+class ChangePasswordForm(CustomForm):
+    current_password = forms.CharField(widget=forms.PasswordInput)
+    confirm_password = forms.CharField(widget=forms.PasswordInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+        widgets = {
+            "password": forms.PasswordInput(),
+        }
+
+    def clean(self):
+        """Make sure passwords match and are valid"""
+        current_password = self.data.get("current_password")
+        if not self.instance.check_password(current_password):
+            self.add_error("current_password", _("Incorrect password"))
+
+        cleaned_data = super().clean()
+        new_password = cleaned_data.get("password")
+        confirm_password = self.data.get("confirm_password")
+
+        if new_password != confirm_password:
+            self.add_error("confirm_password", _("Password does not match"))
+
+        try:
+            validate_password(new_password)
+        except ValidationError as err:
+            self.add_error("password", err)
+
+
+class ConfirmPasswordForm(CustomForm):
+    password = forms.CharField(widget=forms.PasswordInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+        widgets = {
+            "password": forms.PasswordInput(),
+        }
+
+    def clean(self):
+        """Make sure password is correct"""
+        password = self.data.get("password")
+
+        if not self.instance.check_password(password):
+            self.add_error("password", _("Incorrect Password"))

bookwyrm/forms/forms.py

@@ -1,4 +1,5 @@
 """ using django model forms """
+import datetime
 from django import forms
 from django.forms import widgets
 from django.utils.translation import gettext_lazy as _
@@ -7,7 +8,6 @@ from bookwyrm import models
 from bookwyrm.models.user import FeedFilterChoices
 from .custom_form import CustomForm
 
-
 # pylint: disable=missing-class-docstring
 class FeedStatusTypesForm(CustomForm):
     class Meta:
@@ -25,6 +25,10 @@ class ImportForm(forms.Form):
     csv_file = forms.FileField()
 
 
+class ImportUserForm(forms.Form):
+    archive_file = forms.FileField()
+
+
 class ShelfForm(CustomForm):
     class Meta:
         model = models.Shelf
@@ -58,6 +62,21 @@ class ReadThroughForm(CustomForm):
             self.add_error(
                 "stopped_date", _("Reading stopped date cannot be before start date.")
             )
+        current_time = datetime.datetime.now()
+        if (
+            stopped_date is not None
+            and current_time.timestamp() < stopped_date.timestamp()
+        ):
+            self.add_error(
+                "stopped_date", _("Reading stopped date cannot be in the future.")
+            )
+        if (
+            finish_date is not None
+            and current_time.timestamp() < finish_date.timestamp()
+        ):
+            self.add_error(
+                "finish_date", _("Reading finished date cannot be in the future.")
+            )
 
     class Meta:
         model = models.ReadThrough

bookwyrm/forms/landing.py

@@ -1,8 +1,14 @@
 """ Forms for the landing pages """
-from django.forms import PasswordInput
+from django import forms
+from django.contrib.auth.password_validation import validate_password
+from django.core.exceptions import ValidationError
 from django.utils.translation import gettext_lazy as _
 
+import pyotp
+
 from bookwyrm import models
+from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import TWO_FACTOR_LOGIN_VALIDITY_WINDOW
 from .custom_form import CustomForm
 
 
@@ -13,21 +19,40 @@ class LoginForm(CustomForm):
         fields = ["localname", "password"]
         help_texts = {f: None for f in fields}
         widgets = {
-            "password": PasswordInput(),
+            "password": forms.PasswordInput(),
         }
 
+    def infer_username(self):
+        """Users may enter their localname, username, or email"""
+        localname = self.data.get("localname")
+        if "@" in localname:  # looks like an email address to me
+            try:
+                return models.User.objects.get(email=localname).username
+            except models.User.DoesNotExist:  # maybe it's a full username?
+                return localname
+        return f"{localname}@{DOMAIN}"
+
+    def add_invalid_password_error(self):
+        """We don't want to be too specific about this"""
+        # pylint: disable=attribute-defined-outside-init
+        self.non_field_errors = _("Username or password are incorrect")
+
 
 class RegisterForm(CustomForm):
     class Meta:
         model = models.User
         fields = ["localname", "email", "password"]
         help_texts = {f: None for f in fields}
-        widgets = {"password": PasswordInput()}
+        widgets = {"password": forms.PasswordInput()}
 
     def clean(self):
         """Check if the username is taken"""
         cleaned_data = super().clean()
         localname = cleaned_data.get("localname").strip()
+        try:
+            validate_password(cleaned_data.get("password"))
+        except ValidationError as err:
+            self.add_error("password", err)
         if models.User.objects.filter(localname=localname).first():
             self.add_error("localname", _("User with this username already exists"))
 
@@ -43,3 +68,65 @@ class InviteRequestForm(CustomForm):
     class Meta:
         model = models.InviteRequest
         fields = ["email", "answer"]
+
+
+class PasswordResetForm(CustomForm):
+    confirm_password = forms.CharField(widget=forms.PasswordInput)
+
+    class Meta:
+        model = models.User
+        fields = ["password"]
+        widgets = {
+            "password": forms.PasswordInput(),
+        }
+
+    def clean(self):
+        """Make sure the passwords match and are valid"""
+        cleaned_data = super().clean()
+        new_password = cleaned_data.get("password")
+        confirm_password = self.data.get("confirm_password")
+
+        if new_password != confirm_password:
+            self.add_error("confirm_password", _("Password does not match"))
+
+        try:
+            validate_password(new_password)
+        except ValidationError as err:
+            self.add_error("password", err)
+
+
+class Confirm2FAForm(CustomForm):
+    otp = forms.CharField(
+        max_length=6, min_length=6, widget=forms.TextInput(attrs={"autofocus": True})
+    )
+
+    class Meta:
+        model = models.User
+        fields = ["otp_secret", "hotp_count"]
+
+    def clean_otp(self):
+        """Check otp matches"""
+        otp = self.data.get("otp")
+        totp = pyotp.TOTP(self.instance.otp_secret)
+
+        if not totp.verify(otp, valid_window=TWO_FACTOR_LOGIN_VALIDITY_WINDOW):
+
+            if self.instance.hotp_secret:
+                # maybe it's a backup code?
+                hotp = pyotp.HOTP(self.instance.hotp_secret)
+                hotp_count = (
+                    self.instance.hotp_count
+                    if self.instance.hotp_count is not None
+                    else 0
+                )
+
+                if not hotp.verify(otp, hotp_count):
+                    self.add_error("otp", _("Incorrect code"))
+
+                # increment the user hotp_count
+                else:
+                    self.instance.hotp_count = hotp_count + 1
+                    self.instance.save(broadcast=False, update_fields=["hotp_count"])
+
+            else:
+                self.add_error("otp", _("Incorrect code"))

bookwyrm/forms/links.py

@@ -1,4 +1,5 @@
 """ using django model forms """
+
 from urllib.parse import urlparse
 
 from django.utils.translation import gettext_lazy as _
@@ -25,7 +26,7 @@ class FileLinkForm(CustomForm):
         url = cleaned_data.get("url")
         filetype = cleaned_data.get("filetype")
         book = cleaned_data.get("book")
-        domain = urlparse(url).netloc
+        domain = urlparse(url).hostname
         if models.LinkDomain.objects.filter(domain=domain).exists():
             status = models.LinkDomain.objects.get(domain=domain).status
             if status == "blocked":
@@ -36,13 +37,15 @@ class FileLinkForm(CustomForm):
                         "This domain is blocked. Please contact your administrator if you think this is an error."
                     ),
                 )
-            elif models.FileLink.objects.filter(
-                url=url, book=book, filetype=filetype
-            ).exists():
-                # pylint: disable=line-too-long
-                self.add_error(
-                    "url",
-                    _(
-                        "This link with file type has already been added for this book. If it is not visible, the domain is still pending."
-                    ),
-                )
+        if (
+            models.FileLink.objects.filter(url=url, book=book, filetype=filetype)
+            .exclude(pk=self.instance)
+            .exists()
+        ):
+            # pylint: disable=line-too-long
+            self.add_error(
+                "url",
+                _(
+                    "This link with file type has already been added for this book. If it is not visible, the domain is still pending."
+                ),
+            )

bookwyrm/forms/lists.py

@@ -24,7 +24,7 @@ class SortListForm(forms.Form):
     sort_by = ChoiceField(
         choices=(
             ("order", _("List Order")),
-            ("title", _("Book Title")),
+            ("sort_title", _("Book Title")),
             ("rating", _("Rating")),
         ),
         label=_("Sort By"),

bookwyrm/forms/status.py

@@ -53,6 +53,7 @@ class QuotationForm(CustomForm):
             "sensitive",
             "privacy",
             "position",
+            "endposition",
             "position_mode",
         ]
 

bookwyrm/importers/__init__.py

@@ -1,6 +1,7 @@
 """ import classes """
 
 from .importer import Importer
+from .bookwyrm_import import BookwyrmImporter
 from .calibre_import import CalibreImporter
 from .goodreads_import import GoodreadsImporter
 from .librarything_import import LibrarythingImporter

bookwyrm/importers/bookwyrm_import.py

@@ -0,0 +1,24 @@
+"""Import data from Bookwyrm export files"""
+from django.http import QueryDict
+
+from bookwyrm.models import User
+from bookwyrm.models.bookwyrm_import_job import BookwyrmImportJob
+
+
+class BookwyrmImporter:
+    """Import a Bookwyrm User export file.
+    This is kind of a combination of an importer and a connector.
+    """
+
+    # pylint: disable=no-self-use
+    def process_import(
+        self, user: User, archive_file: bytes, settings: QueryDict
+    ) -> BookwyrmImportJob:
+        """import user data from a Bookwyrm export file"""
+
+        required = [k for k in settings if settings.get(k) == "on"]
+
+        job = BookwyrmImportJob.objects.create(
+            user=user, archive_file=archive_file, required=required
+        )
+        return job

bookwyrm/importers/calibre_import.py

@@ -1,4 +1,6 @@
 """ handle reading a csv from calibre """
+from typing import Any, Optional
+
 from bookwyrm.models import Shelf
 
 from . import Importer
@@ -9,7 +11,7 @@ class CalibreImporter(Importer):
 
     service = "Calibre"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         # Add timestamp to row_mappings_guesses for date_added to avoid
         # integrity error
         row_mappings_guesses = []
@@ -23,6 +25,6 @@ class CalibreImporter(Importer):
         self.row_mappings_guesses = row_mappings_guesses
         super().__init__(*args, **kwargs)
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         # Calibre export does not indicate which shelf to use. Use a default one for now
         return Shelf.TO_READ

bookwyrm/importers/importer.py

@@ -1,15 +1,10 @@
 """ handle reading a csv from an external service, defaults are from Goodreads """
 import csv
-import logging
+from datetime import timedelta
+from typing import Iterable, Optional
 
 from django.utils import timezone
-from django.utils.translation import gettext_lazy as _
-
-from bookwyrm import models
-from bookwyrm.models import ImportJob, ImportItem
-from bookwyrm.tasks import app, LOW
-
-logger = logging.getLogger(__name__)
+from bookwyrm.models import ImportJob, ImportItem, SiteSettings, User
 
 
 class Importer:
@@ -24,8 +19,8 @@ class Importer:
         ("id", ["id", "book id"]),
         ("title", ["title"]),
         ("authors", ["author", "authors", "primary author"]),
-        ("isbn_10", ["isbn10", "isbn"]),
-        ("isbn_13", ["isbn13", "isbn", "isbns"]),
+        ("isbn_10", ["isbn10", "isbn", "isbn/uid"]),
+        ("isbn_13", ["isbn13", "isbn", "isbns", "isbn/uid"]),
         ("shelf", ["shelf", "exclusive shelf", "read status", "bookshelf"]),
         ("review_name", ["review name"]),
         ("review_body", ["my review", "review"]),
@@ -41,26 +36,48 @@ class Importer:
         "reading": ["currently-reading", "reading", "currently reading"],
     }
 
-    def create_job(self, user, csv_file, include_reviews, privacy):
+    # pylint: disable=too-many-locals
+    def create_job(
+        self, user: User, csv_file: Iterable[str], include_reviews: bool, privacy: str
+    ) -> ImportJob:
         """check over a csv and creates a database entry for the job"""
         csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter)
-        rows = enumerate(list(csv_reader))
+        rows = list(csv_reader)
+        if len(rows) < 1:
+            raise ValueError("CSV file is empty")
+
+        mappings = (
+            self.create_row_mappings(list(fieldnames))
+            if (fieldnames := csv_reader.fieldnames)
+            else {}
+        )
+
         job = ImportJob.objects.create(
             user=user,
             include_reviews=include_reviews,
             privacy=privacy,
-            mappings=self.create_row_mappings(csv_reader.fieldnames),
+            mappings=mappings,
             source=self.service,
         )
 
-        for index, entry in rows:
+        enforce_limit, allowed_imports = self.get_import_limit(user)
+        if enforce_limit and allowed_imports <= 0:
+            job.complete_job()
+            return job
+        for index, entry in enumerate(rows):
+            if enforce_limit and index >= allowed_imports:
+                break
             self.create_item(job, index, entry)
         return job
 
-    def update_legacy_job(self, job):
+    def update_legacy_job(self, job: ImportJob) -> None:
         """patch up a job that was in the old format"""
         items = job.items
-        headers = list(items.first().data.keys())
+        first_item = items.first()
+        if first_item is None:
+            return
+
+        headers = list(first_item.data.keys())
         job.mappings = self.create_row_mappings(headers)
         job.updated_date = timezone.now()
         job.save()
@@ -71,24 +88,24 @@ class Importer:
             item.normalized_data = normalized
             item.save()
 
-    def create_row_mappings(self, headers):
+    def create_row_mappings(self, headers: list[str]) -> dict[str, Optional[str]]:
         """guess what the headers mean"""
         mappings = {}
         for (key, guesses) in self.row_mappings_guesses:
-            value = [h for h in headers if h.lower() in guesses]
-            value = value[0] if len(value) else None
+            values = [h for h in headers if h.lower() in guesses]
+            value = values[0] if len(values) else None
             if value:
                 headers.remove(value)
             mappings[key] = value
         return mappings
 
-    def create_item(self, job, index, data):
+    def create_item(self, job: ImportJob, index: int, data: dict[str, str]) -> None:
         """creates and saves an import item"""
         normalized = self.normalize_row(data, job.mappings)
         normalized["shelf"] = self.get_shelf(normalized)
         ImportItem(job=job, index=index, data=data, normalized_data=normalized).save()
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         """determine which shelf to use"""
         shelf_name = normalized_row.get("shelf")
         if not shelf_name:
@@ -99,11 +116,35 @@ class Importer:
         ]
         return shelf[0] if shelf else None
 
-    def normalize_row(self, entry, mappings):  # pylint: disable=no-self-use
+    # pylint: disable=no-self-use
+    def normalize_row(
+        self, entry: dict[str, str], mappings: dict[str, Optional[str]]
+    ) -> dict[str, Optional[str]]:
         """use the dataclass to create the formatted row of data"""
-        return {k: entry.get(v) for k, v in mappings.items()}
+        return {k: entry.get(v) if v else None for k, v in mappings.items()}
 
-    def create_retry_job(self, user, original_job, items):
+    # pylint: disable=no-self-use
+    def get_import_limit(self, user: User) -> tuple[int, int]:
+        """check if import limit is set and return how many imports are left"""
+        site_settings = SiteSettings.objects.get()
+        import_size_limit = site_settings.import_size_limit
+        import_limit_reset = site_settings.import_limit_reset
+        enforce_limit = import_size_limit and import_limit_reset
+        allowed_imports = 0
+
+        if enforce_limit:
+            time_range = timezone.now() - timedelta(days=import_limit_reset)
+            import_jobs = ImportJob.objects.filter(
+                user=user, created_date__gte=time_range
+            )
+            # pylint: disable=consider-using-generator
+            imported_books = sum([job.successful_item_count for job in import_jobs])
+            allowed_imports = import_size_limit - imported_books
+        return enforce_limit, allowed_imports
+
+    def create_retry_job(
+        self, user: User, original_job: ImportJob, items: list[ImportItem]
+    ) -> ImportJob:
         """retry items that didn't import"""
         job = ImportJob.objects.create(
             user=user,
@@ -114,131 +155,13 @@ class Importer:
             mappings=original_job.mappings,
             retry=True,
         )
-        for item in items:
+        enforce_limit, allowed_imports = self.get_import_limit(user)
+        if enforce_limit and allowed_imports <= 0:
+            job.complete_job()
+            return job
+        for index, item in enumerate(items):
+            if enforce_limit and index >= allowed_imports:
+                break
             # this will re-normalize the raw data
             self.create_item(job, item.index, item.data)
         return job
-
-    def start_import(self, job):  # pylint: disable=no-self-use
-        """initalizes a csv import job"""
-        result = start_import_task.delay(job.id)
-        job.task_id = result.id
-        job.save()
-
-
-@app.task(queue="low_priority")
-def start_import_task(job_id):
-    """trigger the child tasks for each row"""
-    job = ImportJob.objects.get(id=job_id)
-    # these are sub-tasks so that one big task doesn't use up all the memory in celery
-    for item in job.items.values_list("id", flat=True).all():
-        import_item_task.delay(item)
-
-
-@app.task(queue="low_priority")
-def import_item_task(item_id):
-    """resolve a row into a book"""
-    item = models.ImportItem.objects.get(id=item_id)
-    try:
-        item.resolve()
-    except Exception as err:  # pylint: disable=broad-except
-        item.fail_reason = _("Error loading book")
-        item.save()
-        item.update_job()
-        raise err
-
-    if item.book:
-        # shelves book and handles reviews
-        handle_imported_book(item)
-    else:
-        item.fail_reason = _("Could not find a match for book")
-
-    item.save()
-    item.update_job()
-
-
-def handle_imported_book(item):
-    """process a csv and then post about it"""
-    job = item.job
-    user = job.user
-    if isinstance(item.book, models.Work):
-        item.book = item.book.default_edition
-    if not item.book:
-        item.fail_reason = _("Error loading book")
-        item.save()
-        return
-    if not isinstance(item.book, models.Edition):
-        item.book = item.book.edition
-
-    existing_shelf = models.ShelfBook.objects.filter(book=item.book, user=user).exists()
-
-    # shelve the book if it hasn't been shelved already
-    if item.shelf and not existing_shelf:
-        desired_shelf = models.Shelf.objects.get(identifier=item.shelf, user=user)
-        shelved_date = item.date_added or timezone.now()
-        models.ShelfBook(
-            book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date
-        ).save(priority=LOW)
-
-    for read in item.reads:
-        # check for an existing readthrough with the same dates
-        if models.ReadThrough.objects.filter(
-            user=user,
-            book=item.book,
-            start_date=read.start_date,
-            finish_date=read.finish_date,
-        ).exists():
-            continue
-        read.book = item.book
-        read.user = user
-        read.save()
-
-    if job.include_reviews and (item.rating or item.review) and not item.linked_review:
-        # we don't know the publication date of the review,
-        # but "now" is a bad guess
-        published_date_guess = item.date_read or item.date_added
-        if item.review:
-            # pylint: disable=consider-using-f-string
-            review_title = "Review of {!r} on {!r}".format(
-                item.book.title,
-                job.source,
-            )
-            review = models.Review.objects.filter(
-                user=user,
-                book=item.book,
-                name=review_title,
-                rating=item.rating,
-                published_date=published_date_guess,
-            ).first()
-            if not review:
-                review = models.Review(
-                    user=user,
-                    book=item.book,
-                    name=review_title,
-                    content=item.review,
-                    rating=item.rating,
-                    published_date=published_date_guess,
-                    privacy=job.privacy,
-                )
-                review.save(software="bookwyrm", priority=LOW)
-        else:
-            # just a rating
-            review = models.ReviewRating.objects.filter(
-                user=user,
-                book=item.book,
-                published_date=published_date_guess,
-                rating=item.rating,
-            ).first()
-            if not review:
-                review = models.ReviewRating(
-                    user=user,
-                    book=item.book,
-                    rating=item.rating,
-                    published_date=published_date_guess,
-                    privacy=job.privacy,
-                )
-                review.save(software="bookwyrm", priority=LOW)
-
-        # only broadcast this review to other bookwyrm instances
-        item.linked_review = review
-    item.save()

bookwyrm/importers/librarything_import.py

@@ -1,11 +1,16 @@
 """ handle reading a tsv from librarything """
 import re
+from typing import Optional
 
 from bookwyrm.models import Shelf
 
 from . import Importer
 
 
+def _remove_brackets(value: Optional[str]) -> Optional[str]:
+    return re.sub(r"\[|\]", "", value) if value else None
+
+
 class LibrarythingImporter(Importer):
     """csv downloads from librarything"""
 
@@ -13,16 +18,19 @@ class LibrarythingImporter(Importer):
     delimiter = "\t"
     encoding = "ISO-8859-1"
 
-    def normalize_row(self, entry, mappings):  # pylint: disable=no-self-use
+    def normalize_row(
+        self, entry: dict[str, str], mappings: dict[str, Optional[str]]
+    ) -> dict[str, Optional[str]]:  # pylint: disable=no-self-use
         """use the dataclass to create the formatted row of data"""
-        remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None
-        normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()}
-        isbn_13 = normalized.get("isbn_13")
-        isbn_13 = isbn_13.split(", ") if isbn_13 else []
-        normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 0 else None
+        normalized = {
+            k: _remove_brackets(entry.get(v) if v else None)
+            for k, v in mappings.items()
+        }
+        isbn_13 = value.split(", ") if (value := normalized.get("isbn_13")) else []
+        normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 1 else None
         return normalized
 
-    def get_shelf(self, normalized_row):
+    def get_shelf(self, normalized_row: dict[str, Optional[str]]) -> Optional[str]:
         if normalized_row["date_finished"]:
             return Shelf.READ_FINISHED
         if normalized_row["date_started"]:

bookwyrm/importers/openlibrary_import.py

@@ -1,4 +1,6 @@
 """ handle reading a csv from openlibrary"""
+from typing import Any
+
 from . import Importer
 
 
@@ -7,7 +9,7 @@ class OpenLibraryImporter(Importer):
 
     service = "OpenLibrary"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         self.row_mappings_guesses.append(("openlibrary_key", ["edition id"]))
         self.row_mappings_guesses.append(("openlibrary_work_key", ["work id"]))
         super().__init__(*args, **kwargs)

bookwyrm/isbn/isbn.py

@@ -0,0 +1,128 @@
+""" Use the range message from isbn-international to hyphenate ISBNs """
+import os
+from typing import Optional
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+
+import requests
+
+from bookwyrm import settings
+
+
+def _get_rules(element: Element) -> list[Element]:
+    if (rules_el := element.find("Rules")) is not None:
+        return rules_el.findall("Rule")
+    return []
+
+
+class IsbnHyphenator:
+    """Class to manage the range message xml file and use it to hyphenate ISBNs"""
+
+    __range_message_url = "https://www.isbn-international.org/export_rangemessage.xml"
+    __range_file_path = os.path.join(
+        settings.BASE_DIR, "bookwyrm", "isbn", "RangeMessage.xml"
+    )
+    __element_tree = None
+
+    def update_range_message(self) -> None:
+        """Download the range message xml file and save it locally"""
+        response = requests.get(self.__range_message_url, timeout=15)
+        with open(self.__range_file_path, "w", encoding="utf-8") as file:
+            file.write(response.text)
+        self.__element_tree = None
+
+    def hyphenate(self, isbn_13: Optional[str]) -> Optional[str]:
+        """hyphenate the given ISBN-13 number using the range message"""
+        if isbn_13 is None:
+            return None
+
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        gs1_prefix = isbn_13[:3]
+        try:
+            reg_group = self.__find_reg_group(isbn_13, gs1_prefix)
+        except ValueError:
+            # if the reg groups are invalid, just return the original isbn
+            return isbn_13
+
+        if reg_group is None:
+            return isbn_13  # failed to hyphenate
+
+        registrant = self.__find_registrant(isbn_13, gs1_prefix, reg_group)
+        if registrant is None:
+            return isbn_13  # failed to hyphenate
+
+        publication = isbn_13[len(gs1_prefix) + len(reg_group) + len(registrant) : -1]
+        check_digit = isbn_13[-1:]
+        return "-".join((gs1_prefix, reg_group, registrant, publication, check_digit))
+
+    def __find_reg_group(self, isbn_13: str, gs1_prefix: str) -> Optional[str]:
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        ucc_prefixes_el = self.__element_tree.find("EAN.UCCPrefixes")
+        if ucc_prefixes_el is None:
+            return None
+
+        for ean_ucc_el in ucc_prefixes_el.findall("EAN.UCC"):
+            if (
+                prefix_el := ean_ucc_el.find("Prefix")
+            ) is not None and prefix_el.text == gs1_prefix:
+                for rule_el in _get_rules(ean_ucc_el):
+                    length_el = rule_el.find("Length")
+                    if length_el is None:
+                        continue
+                    length = int(text) if (text := length_el.text) else 0
+                    if length == 0:
+                        continue
+
+                    range_el = rule_el.find("Range")
+                    if range_el is None or range_el.text is None:
+                        continue
+
+                    reg_grp_range = [int(x[:length]) for x in range_el.text.split("-")]
+                    reg_group = isbn_13[len(gs1_prefix) : len(gs1_prefix) + length]
+                    if reg_grp_range[0] <= int(reg_group) <= reg_grp_range[1]:
+                        return reg_group
+                return None
+        return None
+
+    def __find_registrant(
+        self, isbn_13: str, gs1_prefix: str, reg_group: str
+    ) -> Optional[str]:
+        from_ind = len(gs1_prefix) + len(reg_group)
+
+        if self.__element_tree is None:
+            self.__element_tree = ElementTree.parse(self.__range_file_path)
+
+        reg_groups_el = self.__element_tree.find("RegistrationGroups")
+        if reg_groups_el is None:
+            return None
+
+        for group_el in reg_groups_el.findall("Group"):
+            if (
+                prefix_el := group_el.find("Prefix")
+            ) is not None and prefix_el.text == "-".join((gs1_prefix, reg_group)):
+                for rule_el in _get_rules(group_el):
+                    length_el = rule_el.find("Length")
+                    if length_el is None:
+                        continue
+                    length = int(text) if (text := length_el.text) else 0
+                    if length == 0:
+                        continue
+
+                    range_el = rule_el.find("Range")
+                    if range_el is None or range_el.text is None:
+                        continue
+                    registrant_range = [
+                        int(x[:length]) for x in range_el.text.split("-")
+                    ]
+                    registrant = isbn_13[from_ind : from_ind + length]
+                    if registrant_range[0] <= int(registrant) <= registrant_range[1]:
+                        return registrant
+                return None
+        return None
+
+
+hyphenator_singleton = IsbnHyphenator()

bookwyrm/lists_stream.py

@@ -5,7 +5,7 @@ from django.db.models import signals, Count, Q
 
 from bookwyrm import models
 from bookwyrm.redis_store import RedisStore
-from bookwyrm.tasks import app, MEDIUM, HIGH
+from bookwyrm.tasks import app, LISTS
 
 
 class ListsStream(RedisStore):
@@ -24,8 +24,7 @@ class ListsStream(RedisStore):
 
     def add_list(self, book_list):
         """add a list to users' feeds"""
-        # the pipeline contains all the add-to-stream activities
-        self.add_object_to_related_stores(book_list)
+        self.add_object_to_stores(book_list, self.get_stores_for_object(book_list))
 
     def add_user_lists(self, viewer, user):
         """add a user's lists to another user's feed"""
@@ -86,18 +85,19 @@ class ListsStream(RedisStore):
             if group:
                 audience = audience.filter(
                     Q(id=book_list.user.id)  # if the user is the list's owner
-                    | Q(following=book_list.user)  # if the user is following the pwmer
+                    | Q(following=book_list.user)  # if the user is following the owner
                     # if a user is in the group
                     | Q(memberships__group__id=book_list.group.id)
                 )
             else:
                 audience = audience.filter(
                     Q(id=book_list.user.id)  # if the user is the list's owner
-                    | Q(following=book_list.user)  # if the user is following the pwmer
+                    | Q(following=book_list.user)  # if the user is following the owner
                 )
         return audience.distinct()
 
     def get_stores_for_object(self, obj):
+        """the stores that an object belongs in"""
         return [self.stream_id(u) for u in self.get_audience(obj)]
 
     def get_lists_for_user(self, user):  # pylint: disable=no-self-use
@@ -217,14 +217,14 @@ def add_list_on_account_create_command(user_id):
 
 
 # ---- TASKS
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def populate_lists_task(user_id):
     """background task for populating an empty list stream"""
     user = models.User.objects.get(id=user_id)
     ListsStream().populate_lists(user)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def remove_list_task(list_id, re_add=False):
     """remove a list from any stream it might be in"""
     stores = models.User.objects.filter(local=True, is_active=True).values_list(
@@ -233,20 +233,20 @@ def remove_list_task(list_id, re_add=False):
 
     # delete for every store
     stores = [ListsStream().stream_id(idx) for idx in stores]
-    ListsStream().remove_object_from_related_stores(list_id, stores=stores)
+    ListsStream().remove_object_from_stores(list_id, stores)
 
     if re_add:
         add_list_task.delay(list_id)
 
 
-@app.task(queue=HIGH)
+@app.task(queue=LISTS)
 def add_list_task(list_id):
     """add a list to any stream it should be in"""
     book_list = models.List.objects.get(id=list_id)
     ListsStream().add_list(book_list)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
     """remove all lists by a user from a viewer's stream"""
     viewer = models.User.objects.get(id=viewer_id)
@@ -254,7 +254,7 @@ def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
     ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy)
 
 
-@app.task(queue=MEDIUM)
+@app.task(queue=LISTS)
 def add_user_lists_task(viewer_id, user_id):
     """add all lists by a user to a viewer's stream"""
     viewer = models.User.objects.get(id=viewer_id)

bookwyrm/management/commands/compile_themes.py

@@ -0,0 +1,48 @@
+""" Our own command to all scss themes """
+import glob
+import os
+
+import sass
+
+from django.core.management.base import BaseCommand
+
+from sass_processor.apps import APPS_INCLUDE_DIRS
+from sass_processor.processor import SassProcessor
+from sass_processor.utils import get_custom_functions
+
+from bookwyrm import settings
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "SCSS compile all BookWyrm themes"
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """compile"""
+        themes_dir = os.path.join(
+            settings.BASE_DIR, "bookwyrm", "static", "css", "themes", "*.scss"
+        )
+        for theme_scss in glob.glob(themes_dir):
+            basename, _ = os.path.splitext(theme_scss)
+            theme_css = f"{basename}.css"
+            self.compile_sass(theme_scss, theme_css)
+
+    def compile_sass(self, sass_path, css_path):
+        compile_kwargs = {
+            "filename": sass_path,
+            "include_paths": SassProcessor.include_paths + APPS_INCLUDE_DIRS,
+            "custom_functions": get_custom_functions(),
+            "precision": getattr(settings, "SASS_PRECISION", 8),
+            "output_style": getattr(
+                settings,
+                "SASS_OUTPUT_STYLE",
+                "nested" if settings.DEBUG else "compressed",
+            ),
+        }
+
+        content = sass.compile(**compile_kwargs)
+        with open(css_path, "w") as f:
+            f.write(content)
+        self.stdout.write("Compiled SASS/SCSS file: '{0}'\n".format(sass_path))

bookwyrm/management/commands/confirm_email.py

@@ -0,0 +1,19 @@
+""" manually confirm e-mail of user """
+from django.core.management.base import BaseCommand
+
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Manually confirm email for user"
+
+    def add_arguments(self, parser):
+        parser.add_argument("username")
+
+    def handle(self, *args, **options):
+        name = options["username"]
+        user = models.User.objects.get(localname=name)
+        user.reactivate()
+        self.stdout.write(self.style.SUCCESS("User's email is now confirmed."))

bookwyrm/management/commands/deduplicate_book_data.py

@@ -1,44 +1,14 @@
 """ PROCEED WITH CAUTION: uses deduplication fields to permanently
 merge book data objects """
+
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from bookwyrm import models
 
 
-def update_related(canonical, obj):
-    """update all the models with fk to the object being removed"""
-    # move related models to canonical
-    related_models = [
-        (r.remote_field.name, r.related_model) for r in canonical._meta.related_objects
-    ]
-    for (related_field, related_model) in related_models:
-        related_objs = related_model.objects.filter(**{related_field: obj})
-        for related_obj in related_objs:
-            print("replacing in", related_model.__name__, related_field, related_obj.id)
-            try:
-                setattr(related_obj, related_field, canonical)
-                related_obj.save()
-            except TypeError:
-                getattr(related_obj, related_field).add(canonical)
-                getattr(related_obj, related_field).remove(obj)
-
-
-def copy_data(canonical, obj):
-    """try to get the most data possible"""
-    for data_field in obj._meta.get_fields():
-        if not hasattr(data_field, "activitypub_field"):
-            continue
-        data_value = getattr(obj, data_field.name)
-        if not data_value:
-            continue
-        if not getattr(canonical, data_field.name):
-            print("setting data field", data_field.name, data_value)
-            setattr(canonical, data_field.name, data_value)
-    canonical.save()
-
-
-def dedupe_model(model):
+def dedupe_model(model, dry_run=False):
     """combine duplicate editions and update related models"""
+    print(f"deduplicating {model.__name__}:")
     fields = model._meta.get_fields()
     dedupe_fields = [
         f for f in fields if hasattr(f, "deduplication_field") and f.deduplication_field
@@ -47,33 +17,42 @@ def dedupe_model(model):
         dupes = (
             model.objects.values(field.name)
             .annotate(Count(field.name))
-            .filter(**{"%s__count__gt" % field.name: 1})
+            .filter(**{f"{field.name}__count__gt": 1})
+            .exclude(**{field.name: ""})
+            .exclude(**{f"{field.name}__isnull": True})
         )
 
         for dupe in dupes:
             value = dupe[field.name]
-            if not value or value == "":
-                continue
             print("----------")
-            print(dupe)
             objs = model.objects.filter(**{field.name: value}).order_by("id")
             canonical = objs.first()
-            print("keeping", canonical.remote_id)
+            action = "would merge" if dry_run else "merging"
+            print(
+                f"{action} into {model.__name__} {canonical.remote_id} based on {field.name} {value}:"
+            )
             for obj in objs[1:]:
-                print(obj.remote_id)
-                copy_data(canonical, obj)
-                update_related(canonical, obj)
-                # remove the outdated entry
-                obj.delete()
+                print(f"- {obj.remote_id}")
+                absorbed_fields = obj.merge_into(canonical, dry_run=dry_run)
+                print(f"  absorbed fields: {absorbed_fields}")
 
 
 class Command(BaseCommand):
-    """dedplucate allllll the book data models"""
+    """deduplicate allllll the book data models"""
 
     help = "merges duplicate book data"
+
+    def add_arguments(self, parser):
+        """add the arguments for this command"""
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
+
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
-        """run deudplications"""
-        dedupe_model(models.Edition)
-        dedupe_model(models.Work)
-        dedupe_model(models.Author)
+        """run deduplications"""
+        dedupe_model(models.Edition, dry_run=options["dry_run"])
+        dedupe_model(models.Work, dry_run=options["dry_run"])
+        dedupe_model(models.Author, dry_run=options["dry_run"])

bookwyrm/management/commands/erase_deleted_user_data.py

@@ -0,0 +1,43 @@
+""" Erase any data stored about deleted users """
+import sys
+from django.core.management.base import BaseCommand, CommandError
+from bookwyrm import models
+from bookwyrm.models.user import erase_user_data
+
+# pylint: disable=missing-function-docstring
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Remove Two Factor Authorisation from user"
+
+    def add_arguments(self, parser):  # pylint: disable=no-self-use
+        parser.add_argument(
+            "--dryrun",
+            action="store_true",
+            help="Preview users to be cleared without altering the database",
+        )
+
+    def handle(self, *args, **options):  # pylint: disable=unused-argument
+
+        # Check for anything fishy
+        bad_state = models.User.objects.filter(is_deleted=True, is_active=True)
+        if bad_state.exists():
+            raise CommandError(
+                f"{bad_state.count()} user(s) marked as both active and deleted"
+            )
+
+        deleted_users = models.User.objects.filter(is_deleted=True)
+        self.stdout.write(f"Found {deleted_users.count()} deleted users")
+        if options["dryrun"]:
+            self.stdout.write("\n".join(u.username for u in deleted_users[:5]))
+            if deleted_users.count() > 5:
+                self.stdout.write("... and more")
+            sys.exit()
+
+        self.stdout.write("Erasing user data:")
+        for user_id in deleted_users.values_list("id", flat=True):
+            erase_user_data.delay(user_id)
+            self.stdout.write(".", ending="")
+
+        self.stdout.write("")
+        self.stdout.write("Tasks created successfully")

bookwyrm/management/commands/erase_streams.py

@@ -4,12 +4,7 @@ import redis
 
 from bookwyrm import settings
 
-r = redis.Redis(
-    host=settings.REDIS_ACTIVITY_HOST,
-    port=settings.REDIS_ACTIVITY_PORT,
-    password=settings.REDIS_ACTIVITY_PASSWORD,
-    db=settings.REDIS_ACTIVITY_DB_INDEX,
-)
+r = redis.from_url(settings.REDIS_ACTIVITY_URL)
 
 
 def erase_streams():

bookwyrm/management/commands/initdb.py

@@ -8,54 +8,64 @@ from bookwyrm import models
 
 def init_groups():
     """permission levels"""
-    groups = ["admin", "moderator", "editor"]
+    groups = ["admin", "owner", "moderator", "editor"]
     for group in groups:
-        Group.objects.create(name=group)
+        Group.objects.get_or_create(name=group)
 
 
 def init_permissions():
     """permission types"""
     permissions = [
+        {
+            "codename": "manage_registration",
+            "name": "allow or prevent user registration",
+            "groups": ["admin"],
+        },
+        {
+            "codename": "system_administration",
+            "name": "technical controls",
+            "groups": ["admin"],
+        },
         {
             "codename": "edit_instance_settings",
             "name": "change the instance info",
-            "groups": ["admin"],
+            "groups": ["admin", "owner"],
         },
         {
             "codename": "set_user_group",
             "name": "change what group a user is in",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "control_federation",
             "name": "control who to federate with",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "create_invites",
             "name": "issue invitations to join",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "moderate_user",
             "name": "deactivate or silence a user",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "moderate_post",
             "name": "delete other users' posts",
-            "groups": ["admin", "moderator"],
+            "groups": ["admin", "owner", "moderator"],
         },
         {
             "codename": "edit_book",
             "name": "edit book info",
-            "groups": ["admin", "moderator", "editor"],
+            "groups": ["admin", "owner", "moderator", "editor"],
         },
     ]
 
     content_type = ContentType.objects.get_for_model(models.User)
     for permission in permissions:
-        permission_obj = Permission.objects.create(
+        permission_obj, _ = Permission.objects.get_or_create(
             codename=permission["codename"],
             name=permission["name"],
             content_type=content_type,
@@ -107,10 +117,12 @@ def init_connectors():
 
 def init_settings():
     """info about the instance"""
+    group_editor = Group.objects.filter(name="editor").first()
     models.SiteSettings.objects.create(
         support_link="https://www.patreon.com/bookwyrm",
         support_title="Patreon",
         install_mode=True,
+        default_user_auth_group=group_editor,
     )
 
 

bookwyrm/management/commands/instance_version.py

@@ -1,54 +0,0 @@
-""" Get your admin code to allow install """
-from django.core.management.base import BaseCommand
-
-from bookwyrm import models
-from bookwyrm.settings import VERSION
-
-
-# pylint: disable=no-self-use
-class Command(BaseCommand):
-    """command-line options"""
-
-    help = "What version is this?"
-
-    def add_arguments(self, parser):
-        """specify which function to run"""
-        parser.add_argument(
-            "--current",
-            action="store_true",
-            help="Version stored in database",
-        )
-        parser.add_argument(
-            "--target",
-            action="store_true",
-            help="Version stored in settings",
-        )
-        parser.add_argument(
-            "--update",
-            action="store_true",
-            help="Update database version",
-        )
-
-    # pylint: disable=unused-argument
-    def handle(self, *args, **options):
-        """execute init"""
-        site = models.SiteSettings.objects.get()
-        current = site.version or "0.0.1"
-        target = VERSION
-        if options.get("current"):
-            print(current)
-            return
-
-        if options.get("target"):
-            print(target)
-            return
-
-        if options.get("update"):
-            site.version = target
-            site.save()
-            return
-
-        if current != target:
-            print(f"{current}/{target}")
-        else:
-            print(current)

bookwyrm/management/commands/merge_authors.py

@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge author data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+    """merges two authors by ID"""
+
+    help = "merges specified authors into one"
+
+    MODEL = models.Author

bookwyrm/management/commands/merge_editions.py

@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge edition data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+    """merges two editions by ID"""
+
+    help = "merges specified editions into one"
+
+    MODEL = models.Edition

bookwyrm/management/commands/merge_works.py

@@ -0,0 +1,12 @@
+""" PROCEED WITH CAUTION: uses deduplication fields to permanently
+merge work data objects """
+from bookwyrm import models
+from bookwyrm.management.merge_command import MergeCommand
+
+
+class Command(MergeCommand):
+    """merges two works by ID"""
+
+    help = "merges specified works into one"
+
+    MODEL = models.Work

bookwyrm/management/commands/remove_2fa.py

@@ -0,0 +1,22 @@
+"""deactivate two factor auth"""
+
+from django.core.management.base import BaseCommand, CommandError
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Remove Two Factor Authorisation from user"
+
+    def add_arguments(self, parser):
+        parser.add_argument("username")
+
+    def handle(self, *args, **options):
+        name = options["username"]
+        user = models.User.objects.get(localname=name)
+        user.two_factor_auth = False
+        user.save(broadcast=False, update_fields=["two_factor_auth"])
+        self.stdout.write(
+            self.style.SUCCESS("Two Factor Authorisation was removed from user")
+        )

bookwyrm/management/commands/remove_editions.py

@@ -33,10 +33,10 @@ def remove_editions():
 
 
 class Command(BaseCommand):
-    """dedplucate allllll the book data models"""
+    """deduplicate allllll the book data models"""
 
     help = "merges duplicate book data"
     # pylint: disable=no-self-use,unused-argument
     def handle(self, *args, **options):
-        """run deudplications"""
+        """run deduplications"""
         remove_editions()

bookwyrm/management/commands/remove_remote_user_preview_images.py

@@ -0,0 +1,40 @@
+""" Remove preview images for remote users """
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+
+from bookwyrm import models, preview_images
+
+
+# pylint: disable=line-too-long
+class Command(BaseCommand):
+    """Remove preview images for remote users"""
+
+    help = "Remove preview images for remote users"
+
+    # pylint: disable=no-self-use,unused-argument
+    def handle(self, *args, **options):
+        """generate preview images"""
+        self.stdout.write(
+            "   | Hello! I will be removing preview images from remote users."
+        )
+        self.stdout.write(
+            "🧑‍🚒 ⎨ This might take quite long if your instance has a lot of remote users."
+        )
+        self.stdout.write("   | ✧ Thank you for your patience ✧")
+
+        users = models.User.objects.filter(local=False).exclude(
+            Q(preview_image="") | Q(preview_image=None)
+        )
+
+        if len(users) > 0:
+            self.stdout.write(
+                f"   → Remote user preview images ({len(users)}): ", ending=""
+            )
+            for user in users:
+                preview_images.remove_user_preview_image_task.delay(user.id)
+                self.stdout.write(".", ending="")
+            self.stdout.write(" OK 🖼")
+        else:
+            self.stdout.write(f"   | There was no remote users with preview images.")
+
+        self.stdout.write("🧑‍🚒 ⎨ I’m all done! ✧ Enjoy ✧")

bookwyrm/management/commands/repair_editions.py

@@ -0,0 +1,21 @@
+""" Repair editions with missing works """
+from django.core.management.base import BaseCommand
+from bookwyrm import models
+
+
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Repairs an edition that is in a broken state"
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """Find and repair broken editions"""
+        # Find broken editions
+        editions = models.Edition.objects.filter(parent_work__isnull=True)
+        self.stdout.write(f"Repairing {editions.count()} edition(s):")
+
+        # Do repair
+        for edition in editions:
+            edition.repair()
+            self.stdout.write(".", ending="")

bookwyrm/management/commands/revoke_preview_image_tasks.py

@@ -0,0 +1,31 @@
+""" Actually let's not generate those preview images  """
+import json
+from django.core.management.base import BaseCommand
+from bookwyrm.tasks import app
+
+
+class Command(BaseCommand):
+    """Find and revoke image tasks"""
+
+    # pylint: disable=unused-argument
+    def handle(self, *args, **options):
+        """revoke nonessential low priority tasks"""
+        types = [
+            "bookwyrm.preview_images.generate_edition_preview_image_task",
+            "bookwyrm.preview_images.generate_user_preview_image_task",
+        ]
+        self.stdout.write("   | Finding tasks of types:")
+        self.stdout.write("\n".join(types))
+        with app.pool.acquire(block=True) as conn:
+            tasks = conn.default_channel.client.lrange("low_priority", 0, -1)
+            self.stdout.write(f"   | Found {len(tasks)} task(s) in low priority queue")
+
+        revoke_ids = []
+        for task in tasks:
+            task_json = json.loads(task)
+            task_type = task_json.get("headers", {}).get("task")
+            if task_type in types:
+                revoke_ids.append(task_json.get("headers", {}).get("id"))
+                self.stdout.write(".", ending="")
+        self.stdout.write(f"\n   | Revoking {len(revoke_ids)} task(s)")
+        app.control.revoke(revoke_ids)

bookwyrm/management/merge_command.py

@@ -0,0 +1,37 @@
+from django.core.management.base import BaseCommand
+
+
+class MergeCommand(BaseCommand):
+    """base class for merge commands"""
+
+    def add_arguments(self, parser):
+        """add the arguments for this command"""
+        parser.add_argument("--canonical", type=int, required=True)
+        parser.add_argument("--other", type=int, required=True)
+        parser.add_argument(
+            "--dry_run",
+            action="store_true",
+            help="don't actually merge, only print what would happen",
+        )
+
+    # pylint: disable=no-self-use,unused-argument
+    def handle(self, *args, **options):
+        """merge the two objects"""
+        model = self.MODEL
+
+        try:
+            canonical = model.objects.get(id=options["canonical"])
+        except model.DoesNotExist:
+            print("canonical book doesn’t exist!")
+            return
+        try:
+            other = model.objects.get(id=options["other"])
+        except model.DoesNotExist:
+            print("other book doesn’t exist!")
+            return
+
+        absorbed_fields = other.merge_into(canonical, dry_run=options["dry_run"])
+
+        action = "would be" if options["dry_run"] else "has been"
+        print(f"{other.remote_id} {action} merged into {canonical.remote_id}")
+        print(f"absorbed fields: {absorbed_fields}")

bookwyrm/middleware/__init__.py

@@ -1,3 +1,4 @@
 """ look at all this nice middleware! """
 from .timezone_middleware import TimezoneMiddleware
 from .ip_middleware import IPBlocklistMiddleware
+from .file_too_big import FileTooBig

bookwyrm/middleware/file_too_big.py

@@ -0,0 +1,30 @@
+"""Middleware to display a custom 413 error page"""
+
+from django.http import HttpResponse
+from django.shortcuts import render
+from django.core.exceptions import RequestDataTooBig
+
+
+class FileTooBig:
+    """Middleware to display a custom page when a
+    RequestDataTooBig exception is thrown"""
+
+    def __init__(self, get_response):
+        """boilerplate __init__ from Django docs"""
+
+        self.get_response = get_response
+
+    def __call__(self, request):
+        """If RequestDataTooBig is thrown, render the 413 error page"""
+
+        try:
+            body = request.body  # pylint: disable=unused-variable
+
+        except RequestDataTooBig:
+
+            rendered = render(request, "413.html")
+            response = HttpResponse(rendered)
+            return response
+
+        response = self.get_response(request)
+        return response

bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py

@@ -1467,7 +1467,7 @@ class Migration(migrations.Migration):
                 (
                     "expiry",
                     models.DateTimeField(
-                        default=bookwyrm.models.site.get_passowrd_reset_expiry
+                        default=bookwyrm.models.site.get_password_reset_expiry
                     ),
                 ),
                 (

bookwyrm/migrations/0101_auto_20210929_1847.py

@@ -6,7 +6,7 @@ from bookwyrm.connectors.abstract_connector import infer_physical_format
 
 
 def infer_format(app_registry, schema_editor):
-    """set the new phsyical format field based on existing format data"""
+    """set the new physical format field based on existing format data"""
     db_alias = schema_editor.connection.alias
 
     editions = (

bookwyrm/migrations/0102_remove_connector_local.py

@@ -5,7 +5,7 @@ from bookwyrm.settings import DOMAIN
 
 
 def remove_self_connector(app_registry, schema_editor):
-    """set the new phsyical format field based on existing format data"""
+    """set the new physical format field based on existing format data"""
     db_alias = schema_editor.connection.alias
     app_registry.get_model("bookwyrm", "Connector").objects.using(db_alias).filter(
         connector_file="self_connector"

bookwyrm/migrations/0122_alter_annualgoal_year.py

@@ -14,6 +14,8 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name="annualgoal",
             name="year",
-            field=models.IntegerField(default=bookwyrm.models.user.get_current_year),
+            field=models.IntegerField(
+                default=bookwyrm.models.annual_goal.get_current_year
+            ),
         ),
     ]

bookwyrm/migrations/0154_alter_user_preferred_language.py

@@ -0,0 +1,40 @@
+# Generated by Django 3.2.14 on 2022-07-15 19:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0153_merge_20220706_2141"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0155_user_show_guided_tour.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.2.14 on 2022-07-09 23:33
+
+from django.db import migrations, models
+
+
+def existing_users_default(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+    user_model = apps.get_model("bookwyrm", "User")
+    user_model.objects.using(db_alias).filter(local=True).update(show_guided_tour=False)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0154_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="show_guided_tour",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.RunPython(existing_users_default, migrations.RunPython.noop),
+    ]

bookwyrm/migrations/0156_alter_user_preferred_language.py

@@ -0,0 +1,41 @@
+# Generated by Django 3.2.14 on 2022-08-02 18:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0155_user_show_guided_tour"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0157_auto_20220909_2338.py

@@ -0,0 +1,647 @@
+# Generated by Django 3.2.15 on 2022-09-09 23:38
+
+import bookwyrm.models.fields
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0156_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="review",
+            name="rating",
+            field=bookwyrm.models.fields.DecimalField(
+                blank=True,
+                decimal_places=2,
+                default=None,
+                max_digits=3,
+                null=True,
+                validators=[
+                    django.core.validators.MinValueValidator(0.5),
+                    django.core.validators.MaxValueValidator(5),
+                ],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_timezone",
+            field=models.CharField(
+                choices=[
+                    ("Africa/Abidjan", "Africa/Abidjan"),
+                    ("Africa/Accra", "Africa/Accra"),
+                    ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+                    ("Africa/Algiers", "Africa/Algiers"),
+                    ("Africa/Asmara", "Africa/Asmara"),
+                    ("Africa/Asmera", "Africa/Asmera"),
+                    ("Africa/Bamako", "Africa/Bamako"),
+                    ("Africa/Bangui", "Africa/Bangui"),
+                    ("Africa/Banjul", "Africa/Banjul"),
+                    ("Africa/Bissau", "Africa/Bissau"),
+                    ("Africa/Blantyre", "Africa/Blantyre"),
+                    ("Africa/Brazzaville", "Africa/Brazzaville"),
+                    ("Africa/Bujumbura", "Africa/Bujumbura"),
+                    ("Africa/Cairo", "Africa/Cairo"),
+                    ("Africa/Casablanca", "Africa/Casablanca"),
+                    ("Africa/Ceuta", "Africa/Ceuta"),
+                    ("Africa/Conakry", "Africa/Conakry"),
+                    ("Africa/Dakar", "Africa/Dakar"),
+                    ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+                    ("Africa/Djibouti", "Africa/Djibouti"),
+                    ("Africa/Douala", "Africa/Douala"),
+                    ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+                    ("Africa/Freetown", "Africa/Freetown"),
+                    ("Africa/Gaborone", "Africa/Gaborone"),
+                    ("Africa/Harare", "Africa/Harare"),
+                    ("Africa/Johannesburg", "Africa/Johannesburg"),
+                    ("Africa/Juba", "Africa/Juba"),
+                    ("Africa/Kampala", "Africa/Kampala"),
+                    ("Africa/Khartoum", "Africa/Khartoum"),
+                    ("Africa/Kigali", "Africa/Kigali"),
+                    ("Africa/Kinshasa", "Africa/Kinshasa"),
+                    ("Africa/Lagos", "Africa/Lagos"),
+                    ("Africa/Libreville", "Africa/Libreville"),
+                    ("Africa/Lome", "Africa/Lome"),
+                    ("Africa/Luanda", "Africa/Luanda"),
+                    ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "Africa/Lusaka"),
+                    ("Africa/Malabo", "Africa/Malabo"),
+                    ("Africa/Maputo", "Africa/Maputo"),
+                    ("Africa/Maseru", "Africa/Maseru"),
+                    ("Africa/Mbabane", "Africa/Mbabane"),
+                    ("Africa/Mogadishu", "Africa/Mogadishu"),
+                    ("Africa/Monrovia", "Africa/Monrovia"),
+                    ("Africa/Nairobi", "Africa/Nairobi"),
+                    ("Africa/Ndjamena", "Africa/Ndjamena"),
+                    ("Africa/Niamey", "Africa/Niamey"),
+                    ("Africa/Nouakchott", "Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+                    ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+                    ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+                    ("Africa/Timbuktu", "Africa/Timbuktu"),
+                    ("Africa/Tripoli", "Africa/Tripoli"),
+                    ("Africa/Tunis", "Africa/Tunis"),
+                    ("Africa/Windhoek", "Africa/Windhoek"),
+                    ("America/Adak", "America/Adak"),
+                    ("America/Anchorage", "America/Anchorage"),
+                    ("America/Anguilla", "America/Anguilla"),
+                    ("America/Antigua", "America/Antigua"),
+                    ("America/Araguaina", "America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "America/Argentina/Buenos_Aires",
+                    ),
+                    ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+                    (
+                        "America/Argentina/ComodRivadavia",
+                        "America/Argentina/ComodRivadavia",
+                    ),
+                    ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+                    ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+                    ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+                    ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "America/Argentina/Rio_Gallegos",
+                    ),
+                    ("America/Argentina/Salta", "America/Argentina/Salta"),
+                    ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+                    ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+                    ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+                    ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+                    ("America/Aruba", "America/Aruba"),
+                    ("America/Asuncion", "America/Asuncion"),
+                    ("America/Atikokan", "America/Atikokan"),
+                    ("America/Atka", "America/Atka"),
+                    ("America/Bahia", "America/Bahia"),
+                    ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+                    ("America/Barbados", "America/Barbados"),
+                    ("America/Belem", "America/Belem"),
+                    ("America/Belize", "America/Belize"),
+                    ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+                    ("America/Boa_Vista", "America/Boa_Vista"),
+                    ("America/Bogota", "America/Bogota"),
+                    ("America/Boise", "America/Boise"),
+                    ("America/Buenos_Aires", "America/Buenos_Aires"),
+                    ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+                    ("America/Campo_Grande", "America/Campo_Grande"),
+                    ("America/Cancun", "America/Cancun"),
+                    ("America/Caracas", "America/Caracas"),
+                    ("America/Catamarca", "America/Catamarca"),
+                    ("America/Cayenne", "America/Cayenne"),
+                    ("America/Cayman", "America/Cayman"),
+                    ("America/Chicago", "America/Chicago"),
+                    ("America/Chihuahua", "America/Chihuahua"),
+                    ("America/Coral_Harbour", "America/Coral_Harbour"),
+                    ("America/Cordoba", "America/Cordoba"),
+                    ("America/Costa_Rica", "America/Costa_Rica"),
+                    ("America/Creston", "America/Creston"),
+                    ("America/Cuiaba", "America/Cuiaba"),
+                    ("America/Curacao", "America/Curacao"),
+                    ("America/Danmarkshavn", "America/Danmarkshavn"),
+                    ("America/Dawson", "America/Dawson"),
+                    ("America/Dawson_Creek", "America/Dawson_Creek"),
+                    ("America/Denver", "America/Denver"),
+                    ("America/Detroit", "America/Detroit"),
+                    ("America/Dominica", "America/Dominica"),
+                    ("America/Edmonton", "America/Edmonton"),
+                    ("America/Eirunepe", "America/Eirunepe"),
+                    ("America/El_Salvador", "America/El_Salvador"),
+                    ("America/Ensenada", "America/Ensenada"),
+                    ("America/Fort_Nelson", "America/Fort_Nelson"),
+                    ("America/Fort_Wayne", "America/Fort_Wayne"),
+                    ("America/Fortaleza", "America/Fortaleza"),
+                    ("America/Glace_Bay", "America/Glace_Bay"),
+                    ("America/Godthab", "America/Godthab"),
+                    ("America/Goose_Bay", "America/Goose_Bay"),
+                    ("America/Grand_Turk", "America/Grand_Turk"),
+                    ("America/Grenada", "America/Grenada"),
+                    ("America/Guadeloupe", "America/Guadeloupe"),
+                    ("America/Guatemala", "America/Guatemala"),
+                    ("America/Guayaquil", "America/Guayaquil"),
+                    ("America/Guyana", "America/Guyana"),
+                    ("America/Halifax", "America/Halifax"),
+                    ("America/Havana", "America/Havana"),
+                    ("America/Hermosillo", "America/Hermosillo"),
+                    ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+                    ("America/Indiana/Knox", "America/Indiana/Knox"),
+                    ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+                    ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+                    ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+                    ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+                    ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+                    ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+                    ("America/Indianapolis", "America/Indianapolis"),
+                    ("America/Inuvik", "America/Inuvik"),
+                    ("America/Iqaluit", "America/Iqaluit"),
+                    ("America/Jamaica", "America/Jamaica"),
+                    ("America/Jujuy", "America/Jujuy"),
+                    ("America/Juneau", "America/Juneau"),
+                    ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+                    ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+                    ("America/Knox_IN", "America/Knox_IN"),
+                    ("America/Kralendijk", "America/Kralendijk"),
+                    ("America/La_Paz", "America/La_Paz"),
+                    ("America/Lima", "America/Lima"),
+                    ("America/Los_Angeles", "America/Los_Angeles"),
+                    ("America/Louisville", "America/Louisville"),
+                    ("America/Lower_Princes", "America/Lower_Princes"),
+                    ("America/Maceio", "America/Maceio"),
+                    ("America/Managua", "America/Managua"),
+                    ("America/Manaus", "America/Manaus"),
+                    ("America/Marigot", "America/Marigot"),
+                    ("America/Martinique", "America/Martinique"),
+                    ("America/Matamoros", "America/Matamoros"),
+                    ("America/Mazatlan", "America/Mazatlan"),
+                    ("America/Mendoza", "America/Mendoza"),
+                    ("America/Menominee", "America/Menominee"),
+                    ("America/Merida", "America/Merida"),
+                    ("America/Metlakatla", "America/Metlakatla"),
+                    ("America/Mexico_City", "America/Mexico_City"),
+                    ("America/Miquelon", "America/Miquelon"),
+                    ("America/Moncton", "America/Moncton"),
+                    ("America/Monterrey", "America/Monterrey"),
+                    ("America/Montevideo", "America/Montevideo"),
+                    ("America/Montreal", "America/Montreal"),
+                    ("America/Montserrat", "America/Montserrat"),
+                    ("America/Nassau", "America/Nassau"),
+                    ("America/New_York", "America/New_York"),
+                    ("America/Nipigon", "America/Nipigon"),
+                    ("America/Nome", "America/Nome"),
+                    ("America/Noronha", "America/Noronha"),
+                    ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+                    ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Nuuk", "America/Nuuk"),
+                    ("America/Ojinaga", "America/Ojinaga"),
+                    ("America/Panama", "America/Panama"),
+                    ("America/Pangnirtung", "America/Pangnirtung"),
+                    ("America/Paramaribo", "America/Paramaribo"),
+                    ("America/Phoenix", "America/Phoenix"),
+                    ("America/Port-au-Prince", "America/Port-au-Prince"),
+                    ("America/Port_of_Spain", "America/Port_of_Spain"),
+                    ("America/Porto_Acre", "America/Porto_Acre"),
+                    ("America/Porto_Velho", "America/Porto_Velho"),
+                    ("America/Puerto_Rico", "America/Puerto_Rico"),
+                    ("America/Punta_Arenas", "America/Punta_Arenas"),
+                    ("America/Rainy_River", "America/Rainy_River"),
+                    ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+                    ("America/Recife", "America/Recife"),
+                    ("America/Regina", "America/Regina"),
+                    ("America/Resolute", "America/Resolute"),
+                    ("America/Rio_Branco", "America/Rio_Branco"),
+                    ("America/Rosario", "America/Rosario"),
+                    ("America/Santa_Isabel", "America/Santa_Isabel"),
+                    ("America/Santarem", "America/Santarem"),
+                    ("America/Santiago", "America/Santiago"),
+                    ("America/Santo_Domingo", "America/Santo_Domingo"),
+                    ("America/Sao_Paulo", "America/Sao_Paulo"),
+                    ("America/Scoresbysund", "America/Scoresbysund"),
+                    ("America/Shiprock", "America/Shiprock"),
+                    ("America/Sitka", "America/Sitka"),
+                    ("America/St_Barthelemy", "America/St_Barthelemy"),
+                    ("America/St_Johns", "America/St_Johns"),
+                    ("America/St_Kitts", "America/St_Kitts"),
+                    ("America/St_Lucia", "America/St_Lucia"),
+                    ("America/St_Thomas", "America/St_Thomas"),
+                    ("America/St_Vincent", "America/St_Vincent"),
+                    ("America/Swift_Current", "America/Swift_Current"),
+                    ("America/Tegucigalpa", "America/Tegucigalpa"),
+                    ("America/Thule", "America/Thule"),
+                    ("America/Thunder_Bay", "America/Thunder_Bay"),
+                    ("America/Tijuana", "America/Tijuana"),
+                    ("America/Toronto", "America/Toronto"),
+                    ("America/Tortola", "America/Tortola"),
+                    ("America/Vancouver", "America/Vancouver"),
+                    ("America/Virgin", "America/Virgin"),
+                    ("America/Whitehorse", "America/Whitehorse"),
+                    ("America/Winnipeg", "America/Winnipeg"),
+                    ("America/Yakutat", "America/Yakutat"),
+                    ("America/Yellowknife", "America/Yellowknife"),
+                    ("Antarctica/Casey", "Antarctica/Casey"),
+                    ("Antarctica/Davis", "Antarctica/Davis"),
+                    ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+                    ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+                    ("Antarctica/Mawson", "Antarctica/Mawson"),
+                    ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+                    ("Antarctica/Palmer", "Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "Antarctica/Rothera"),
+                    ("Antarctica/South_Pole", "Antarctica/South_Pole"),
+                    ("Antarctica/Syowa", "Antarctica/Syowa"),
+                    ("Antarctica/Troll", "Antarctica/Troll"),
+                    ("Antarctica/Vostok", "Antarctica/Vostok"),
+                    ("Arctic/Longyearbyen", "Arctic/Longyearbyen"),
+                    ("Asia/Aden", "Asia/Aden"),
+                    ("Asia/Almaty", "Asia/Almaty"),
+                    ("Asia/Amman", "Asia/Amman"),
+                    ("Asia/Anadyr", "Asia/Anadyr"),
+                    ("Asia/Aqtau", "Asia/Aqtau"),
+                    ("Asia/Aqtobe", "Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "Asia/Ashgabat"),
+                    ("Asia/Ashkhabad", "Asia/Ashkhabad"),
+                    ("Asia/Atyrau", "Asia/Atyrau"),
+                    ("Asia/Baghdad", "Asia/Baghdad"),
+                    ("Asia/Bahrain", "Asia/Bahrain"),
+                    ("Asia/Baku", "Asia/Baku"),
+                    ("Asia/Bangkok", "Asia/Bangkok"),
+                    ("Asia/Barnaul", "Asia/Barnaul"),
+                    ("Asia/Beirut", "Asia/Beirut"),
+                    ("Asia/Bishkek", "Asia/Bishkek"),
+                    ("Asia/Brunei", "Asia/Brunei"),
+                    ("Asia/Calcutta", "Asia/Calcutta"),
+                    ("Asia/Chita", "Asia/Chita"),
+                    ("Asia/Choibalsan", "Asia/Choibalsan"),
+                    ("Asia/Chongqing", "Asia/Chongqing"),
+                    ("Asia/Chungking", "Asia/Chungking"),
+                    ("Asia/Colombo", "Asia/Colombo"),
+                    ("Asia/Dacca", "Asia/Dacca"),
+                    ("Asia/Damascus", "Asia/Damascus"),
+                    ("Asia/Dhaka", "Asia/Dhaka"),
+                    ("Asia/Dili", "Asia/Dili"),
+                    ("Asia/Dubai", "Asia/Dubai"),
+                    ("Asia/Dushanbe", "Asia/Dushanbe"),
+                    ("Asia/Famagusta", "Asia/Famagusta"),
+                    ("Asia/Gaza", "Asia/Gaza"),
+                    ("Asia/Harbin", "Asia/Harbin"),
+                    ("Asia/Hebron", "Asia/Hebron"),
+                    ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+                    ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+                    ("Asia/Hovd", "Asia/Hovd"),
+                    ("Asia/Irkutsk", "Asia/Irkutsk"),
+                    ("Asia/Istanbul", "Asia/Istanbul"),
+                    ("Asia/Jakarta", "Asia/Jakarta"),
+                    ("Asia/Jayapura", "Asia/Jayapura"),
+                    ("Asia/Jerusalem", "Asia/Jerusalem"),
+                    ("Asia/Kabul", "Asia/Kabul"),
+                    ("Asia/Kamchatka", "Asia/Kamchatka"),
+                    ("Asia/Karachi", "Asia/Karachi"),
+                    ("Asia/Kashgar", "Asia/Kashgar"),
+                    ("Asia/Kathmandu", "Asia/Kathmandu"),
+                    ("Asia/Katmandu", "Asia/Katmandu"),
+                    ("Asia/Khandyga", "Asia/Khandyga"),
+                    ("Asia/Kolkata", "Asia/Kolkata"),
+                    ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+                    ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "Asia/Kuching"),
+                    ("Asia/Kuwait", "Asia/Kuwait"),
+                    ("Asia/Macao", "Asia/Macao"),
+                    ("Asia/Macau", "Asia/Macau"),
+                    ("Asia/Magadan", "Asia/Magadan"),
+                    ("Asia/Makassar", "Asia/Makassar"),
+                    ("Asia/Manila", "Asia/Manila"),
+                    ("Asia/Muscat", "Asia/Muscat"),
+                    ("Asia/Nicosia", "Asia/Nicosia"),
+                    ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+                    ("Asia/Omsk", "Asia/Omsk"),
+                    ("Asia/Oral", "Asia/Oral"),
+                    ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "Asia/Pontianak"),
+                    ("Asia/Pyongyang", "Asia/Pyongyang"),
+                    ("Asia/Qatar", "Asia/Qatar"),
+                    ("Asia/Qostanay", "Asia/Qostanay"),
+                    ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+                    ("Asia/Rangoon", "Asia/Rangoon"),
+                    ("Asia/Riyadh", "Asia/Riyadh"),
+                    ("Asia/Saigon", "Asia/Saigon"),
+                    ("Asia/Sakhalin", "Asia/Sakhalin"),
+                    ("Asia/Samarkand", "Asia/Samarkand"),
+                    ("Asia/Seoul", "Asia/Seoul"),
+                    ("Asia/Shanghai", "Asia/Shanghai"),
+                    ("Asia/Singapore", "Asia/Singapore"),
+                    ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+                    ("Asia/Taipei", "Asia/Taipei"),
+                    ("Asia/Tashkent", "Asia/Tashkent"),
+                    ("Asia/Tbilisi", "Asia/Tbilisi"),
+                    ("Asia/Tehran", "Asia/Tehran"),
+                    ("Asia/Tel_Aviv", "Asia/Tel_Aviv"),
+                    ("Asia/Thimbu", "Asia/Thimbu"),
+                    ("Asia/Thimphu", "Asia/Thimphu"),
+                    ("Asia/Tokyo", "Asia/Tokyo"),
+                    ("Asia/Tomsk", "Asia/Tomsk"),
+                    ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"),
+                    ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+                    ("Asia/Ulan_Bator", "Asia/Ulan_Bator"),
+                    ("Asia/Urumqi", "Asia/Urumqi"),
+                    ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+                    ("Asia/Vientiane", "Asia/Vientiane"),
+                    ("Asia/Vladivostok", "Asia/Vladivostok"),
+                    ("Asia/Yakutsk", "Asia/Yakutsk"),
+                    ("Asia/Yangon", "Asia/Yangon"),
+                    ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+                    ("Asia/Yerevan", "Asia/Yerevan"),
+                    ("Atlantic/Azores", "Atlantic/Azores"),
+                    ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+                    ("Atlantic/Canary", "Atlantic/Canary"),
+                    ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+                    ("Atlantic/Faeroe", "Atlantic/Faeroe"),
+                    ("Atlantic/Faroe", "Atlantic/Faroe"),
+                    ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"),
+                    ("Atlantic/Madeira", "Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+                    ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+                    ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+                    ("Atlantic/Stanley", "Atlantic/Stanley"),
+                    ("Australia/ACT", "Australia/ACT"),
+                    ("Australia/Adelaide", "Australia/Adelaide"),
+                    ("Australia/Brisbane", "Australia/Brisbane"),
+                    ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+                    ("Australia/Canberra", "Australia/Canberra"),
+                    ("Australia/Currie", "Australia/Currie"),
+                    ("Australia/Darwin", "Australia/Darwin"),
+                    ("Australia/Eucla", "Australia/Eucla"),
+                    ("Australia/Hobart", "Australia/Hobart"),
+                    ("Australia/LHI", "Australia/LHI"),
+                    ("Australia/Lindeman", "Australia/Lindeman"),
+                    ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "Australia/Melbourne"),
+                    ("Australia/NSW", "Australia/NSW"),
+                    ("Australia/North", "Australia/North"),
+                    ("Australia/Perth", "Australia/Perth"),
+                    ("Australia/Queensland", "Australia/Queensland"),
+                    ("Australia/South", "Australia/South"),
+                    ("Australia/Sydney", "Australia/Sydney"),
+                    ("Australia/Tasmania", "Australia/Tasmania"),
+                    ("Australia/Victoria", "Australia/Victoria"),
+                    ("Australia/West", "Australia/West"),
+                    ("Australia/Yancowinna", "Australia/Yancowinna"),
+                    ("Brazil/Acre", "Brazil/Acre"),
+                    ("Brazil/DeNoronha", "Brazil/DeNoronha"),
+                    ("Brazil/East", "Brazil/East"),
+                    ("Brazil/West", "Brazil/West"),
+                    ("CET", "CET"),
+                    ("CST6CDT", "CST6CDT"),
+                    ("Canada/Atlantic", "Canada/Atlantic"),
+                    ("Canada/Central", "Canada/Central"),
+                    ("Canada/Eastern", "Canada/Eastern"),
+                    ("Canada/Mountain", "Canada/Mountain"),
+                    ("Canada/Newfoundland", "Canada/Newfoundland"),
+                    ("Canada/Pacific", "Canada/Pacific"),
+                    ("Canada/Saskatchewan", "Canada/Saskatchewan"),
+                    ("Canada/Yukon", "Canada/Yukon"),
+                    ("Chile/Continental", "Chile/Continental"),
+                    ("Chile/EasterIsland", "Chile/EasterIsland"),
+                    ("Cuba", "Cuba"),
+                    ("EET", "EET"),
+                    ("EST", "EST"),
+                    ("EST5EDT", "EST5EDT"),
+                    ("Egypt", "Egypt"),
+                    ("Eire", "Eire"),
+                    ("Etc/GMT", "Etc/GMT"),
+                    ("Etc/GMT+0", "Etc/GMT+0"),
+                    ("Etc/GMT+1", "Etc/GMT+1"),
+                    ("Etc/GMT+10", "Etc/GMT+10"),
+                    ("Etc/GMT+11", "Etc/GMT+11"),
+                    ("Etc/GMT+12", "Etc/GMT+12"),
+                    ("Etc/GMT+2", "Etc/GMT+2"),
+                    ("Etc/GMT+3", "Etc/GMT+3"),
+                    ("Etc/GMT+4", "Etc/GMT+4"),
+                    ("Etc/GMT+5", "Etc/GMT+5"),
+                    ("Etc/GMT+6", "Etc/GMT+6"),
+                    ("Etc/GMT+7", "Etc/GMT+7"),
+                    ("Etc/GMT+8", "Etc/GMT+8"),
+                    ("Etc/GMT+9", "Etc/GMT+9"),
+                    ("Etc/GMT-0", "Etc/GMT-0"),
+                    ("Etc/GMT-1", "Etc/GMT-1"),
+                    ("Etc/GMT-10", "Etc/GMT-10"),
+                    ("Etc/GMT-11", "Etc/GMT-11"),
+                    ("Etc/GMT-12", "Etc/GMT-12"),
+                    ("Etc/GMT-13", "Etc/GMT-13"),
+                    ("Etc/GMT-14", "Etc/GMT-14"),
+                    ("Etc/GMT-2", "Etc/GMT-2"),
+                    ("Etc/GMT-3", "Etc/GMT-3"),
+                    ("Etc/GMT-4", "Etc/GMT-4"),
+                    ("Etc/GMT-5", "Etc/GMT-5"),
+                    ("Etc/GMT-6", "Etc/GMT-6"),
+                    ("Etc/GMT-7", "Etc/GMT-7"),
+                    ("Etc/GMT-8", "Etc/GMT-8"),
+                    ("Etc/GMT-9", "Etc/GMT-9"),
+                    ("Etc/GMT0", "Etc/GMT0"),
+                    ("Etc/Greenwich", "Etc/Greenwich"),
+                    ("Etc/UCT", "Etc/UCT"),
+                    ("Etc/UTC", "Etc/UTC"),
+                    ("Etc/Universal", "Etc/Universal"),
+                    ("Etc/Zulu", "Etc/Zulu"),
+                    ("Europe/Amsterdam", "Europe/Amsterdam"),
+                    ("Europe/Andorra", "Europe/Andorra"),
+                    ("Europe/Astrakhan", "Europe/Astrakhan"),
+                    ("Europe/Athens", "Europe/Athens"),
+                    ("Europe/Belfast", "Europe/Belfast"),
+                    ("Europe/Belgrade", "Europe/Belgrade"),
+                    ("Europe/Berlin", "Europe/Berlin"),
+                    ("Europe/Bratislava", "Europe/Bratislava"),
+                    ("Europe/Brussels", "Europe/Brussels"),
+                    ("Europe/Bucharest", "Europe/Bucharest"),
+                    ("Europe/Budapest", "Europe/Budapest"),
+                    ("Europe/Busingen", "Europe/Busingen"),
+                    ("Europe/Chisinau", "Europe/Chisinau"),
+                    ("Europe/Copenhagen", "Europe/Copenhagen"),
+                    ("Europe/Dublin", "Europe/Dublin"),
+                    ("Europe/Gibraltar", "Europe/Gibraltar"),
+                    ("Europe/Guernsey", "Europe/Guernsey"),
+                    ("Europe/Helsinki", "Europe/Helsinki"),
+                    ("Europe/Isle_of_Man", "Europe/Isle_of_Man"),
+                    ("Europe/Istanbul", "Europe/Istanbul"),
+                    ("Europe/Jersey", "Europe/Jersey"),
+                    ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+                    ("Europe/Kiev", "Europe/Kiev"),
+                    ("Europe/Kirov", "Europe/Kirov"),
+                    ("Europe/Kyiv", "Europe/Kyiv"),
+                    ("Europe/Lisbon", "Europe/Lisbon"),
+                    ("Europe/Ljubljana", "Europe/Ljubljana"),
+                    ("Europe/London", "Europe/London"),
+                    ("Europe/Luxembourg", "Europe/Luxembourg"),
+                    ("Europe/Madrid", "Europe/Madrid"),
+                    ("Europe/Malta", "Europe/Malta"),
+                    ("Europe/Mariehamn", "Europe/Mariehamn"),
+                    ("Europe/Minsk", "Europe/Minsk"),
+                    ("Europe/Monaco", "Europe/Monaco"),
+                    ("Europe/Moscow", "Europe/Moscow"),
+                    ("Europe/Nicosia", "Europe/Nicosia"),
+                    ("Europe/Oslo", "Europe/Oslo"),
+                    ("Europe/Paris", "Europe/Paris"),
+                    ("Europe/Podgorica", "Europe/Podgorica"),
+                    ("Europe/Prague", "Europe/Prague"),
+                    ("Europe/Riga", "Europe/Riga"),
+                    ("Europe/Rome", "Europe/Rome"),
+                    ("Europe/Samara", "Europe/Samara"),
+                    ("Europe/San_Marino", "Europe/San_Marino"),
+                    ("Europe/Sarajevo", "Europe/Sarajevo"),
+                    ("Europe/Saratov", "Europe/Saratov"),
+                    ("Europe/Simferopol", "Europe/Simferopol"),
+                    ("Europe/Skopje", "Europe/Skopje"),
+                    ("Europe/Sofia", "Europe/Sofia"),
+                    ("Europe/Stockholm", "Europe/Stockholm"),
+                    ("Europe/Tallinn", "Europe/Tallinn"),
+                    ("Europe/Tirane", "Europe/Tirane"),
+                    ("Europe/Tiraspol", "Europe/Tiraspol"),
+                    ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+                    ("Europe/Uzhgorod", "Europe/Uzhgorod"),
+                    ("Europe/Vaduz", "Europe/Vaduz"),
+                    ("Europe/Vatican", "Europe/Vatican"),
+                    ("Europe/Vienna", "Europe/Vienna"),
+                    ("Europe/Vilnius", "Europe/Vilnius"),
+                    ("Europe/Volgograd", "Europe/Volgograd"),
+                    ("Europe/Warsaw", "Europe/Warsaw"),
+                    ("Europe/Zagreb", "Europe/Zagreb"),
+                    ("Europe/Zaporozhye", "Europe/Zaporozhye"),
+                    ("Europe/Zurich", "Europe/Zurich"),
+                    ("GB", "GB"),
+                    ("GB-Eire", "GB-Eire"),
+                    ("GMT", "GMT"),
+                    ("GMT+0", "GMT+0"),
+                    ("GMT-0", "GMT-0"),
+                    ("GMT0", "GMT0"),
+                    ("Greenwich", "Greenwich"),
+                    ("HST", "HST"),
+                    ("Hongkong", "Hongkong"),
+                    ("Iceland", "Iceland"),
+                    ("Indian/Antananarivo", "Indian/Antananarivo"),
+                    ("Indian/Chagos", "Indian/Chagos"),
+                    ("Indian/Christmas", "Indian/Christmas"),
+                    ("Indian/Cocos", "Indian/Cocos"),
+                    ("Indian/Comoro", "Indian/Comoro"),
+                    ("Indian/Kerguelen", "Indian/Kerguelen"),
+                    ("Indian/Mahe", "Indian/Mahe"),
+                    ("Indian/Maldives", "Indian/Maldives"),
+                    ("Indian/Mauritius", "Indian/Mauritius"),
+                    ("Indian/Mayotte", "Indian/Mayotte"),
+                    ("Indian/Reunion", "Indian/Reunion"),
+                    ("Iran", "Iran"),
+                    ("Israel", "Israel"),
+                    ("Jamaica", "Jamaica"),
+                    ("Japan", "Japan"),
+                    ("Kwajalein", "Kwajalein"),
+                    ("Libya", "Libya"),
+                    ("MET", "MET"),
+                    ("MST", "MST"),
+                    ("MST7MDT", "MST7MDT"),
+                    ("Mexico/BajaNorte", "Mexico/BajaNorte"),
+                    ("Mexico/BajaSur", "Mexico/BajaSur"),
+                    ("Mexico/General", "Mexico/General"),
+                    ("NZ", "NZ"),
+                    ("NZ-CHAT", "NZ-CHAT"),
+                    ("Navajo", "Navajo"),
+                    ("PRC", "PRC"),
+                    ("PST8PDT", "PST8PDT"),
+                    ("Pacific/Apia", "Pacific/Apia"),
+                    ("Pacific/Auckland", "Pacific/Auckland"),
+                    ("Pacific/Bougainville", "Pacific/Bougainville"),
+                    ("Pacific/Chatham", "Pacific/Chatham"),
+                    ("Pacific/Chuuk", "Pacific/Chuuk"),
+                    ("Pacific/Easter", "Pacific/Easter"),
+                    ("Pacific/Efate", "Pacific/Efate"),
+                    ("Pacific/Enderbury", "Pacific/Enderbury"),
+                    ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+                    ("Pacific/Fiji", "Pacific/Fiji"),
+                    ("Pacific/Funafuti", "Pacific/Funafuti"),
+                    ("Pacific/Galapagos", "Pacific/Galapagos"),
+                    ("Pacific/Gambier", "Pacific/Gambier"),
+                    ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+                    ("Pacific/Guam", "Pacific/Guam"),
+                    ("Pacific/Honolulu", "Pacific/Honolulu"),
+                    ("Pacific/Johnston", "Pacific/Johnston"),
+                    ("Pacific/Kanton", "Pacific/Kanton"),
+                    ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+                    ("Pacific/Kosrae", "Pacific/Kosrae"),
+                    ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "Pacific/Majuro"),
+                    ("Pacific/Marquesas", "Pacific/Marquesas"),
+                    ("Pacific/Midway", "Pacific/Midway"),
+                    ("Pacific/Nauru", "Pacific/Nauru"),
+                    ("Pacific/Niue", "Pacific/Niue"),
+                    ("Pacific/Norfolk", "Pacific/Norfolk"),
+                    ("Pacific/Noumea", "Pacific/Noumea"),
+                    ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+                    ("Pacific/Palau", "Pacific/Palau"),
+                    ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+                    ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+                    ("Pacific/Ponape", "Pacific/Ponape"),
+                    ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+                    ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+                    ("Pacific/Saipan", "Pacific/Saipan"),
+                    ("Pacific/Samoa", "Pacific/Samoa"),
+                    ("Pacific/Tahiti", "Pacific/Tahiti"),
+                    ("Pacific/Tarawa", "Pacific/Tarawa"),
+                    ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+                    ("Pacific/Truk", "Pacific/Truk"),
+                    ("Pacific/Wake", "Pacific/Wake"),
+                    ("Pacific/Wallis", "Pacific/Wallis"),
+                    ("Pacific/Yap", "Pacific/Yap"),
+                    ("Poland", "Poland"),
+                    ("Portugal", "Portugal"),
+                    ("ROC", "ROC"),
+                    ("ROK", "ROK"),
+                    ("Singapore", "Singapore"),
+                    ("Turkey", "Turkey"),
+                    ("UCT", "UCT"),
+                    ("US/Alaska", "US/Alaska"),
+                    ("US/Aleutian", "US/Aleutian"),
+                    ("US/Arizona", "US/Arizona"),
+                    ("US/Central", "US/Central"),
+                    ("US/East-Indiana", "US/East-Indiana"),
+                    ("US/Eastern", "US/Eastern"),
+                    ("US/Hawaii", "US/Hawaii"),
+                    ("US/Indiana-Starke", "US/Indiana-Starke"),
+                    ("US/Michigan", "US/Michigan"),
+                    ("US/Mountain", "US/Mountain"),
+                    ("US/Pacific", "US/Pacific"),
+                    ("US/Samoa", "US/Samoa"),
+                    ("UTC", "UTC"),
+                    ("Universal", "Universal"),
+                    ("W-SU", "W-SU"),
+                    ("WET", "WET"),
+                    ("Zulu", "Zulu"),
+                ],
+                default="UTC",
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0158_auto_20220919_1634.py

@@ -0,0 +1,65 @@
+# Generated by Django 3.2.15 on 2022-09-19 16:34
+
+import bookwyrm.models.fields
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0157_auto_20220909_2338"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="automod",
+            name="created_date",
+            field=models.DateTimeField(
+                auto_now_add=True, default=django.utils.timezone.now
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="automod",
+            name="remote_id",
+            field=bookwyrm.models.fields.RemoteIdField(
+                max_length=255,
+                null=True,
+                validators=[bookwyrm.models.fields.validate_remote_id],
+            ),
+        ),
+        migrations.AddField(
+            model_name="automod",
+            name="updated_date",
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name="emailblocklist",
+            name="remote_id",
+            field=bookwyrm.models.fields.RemoteIdField(
+                max_length=255,
+                null=True,
+                validators=[bookwyrm.models.fields.validate_remote_id],
+            ),
+        ),
+        migrations.AddField(
+            model_name="emailblocklist",
+            name="updated_date",
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name="ipblocklist",
+            name="remote_id",
+            field=bookwyrm.models.fields.RemoteIdField(
+                max_length=255,
+                null=True,
+                validators=[bookwyrm.models.fields.validate_remote_id],
+            ),
+        ),
+        migrations.AddField(
+            model_name="ipblocklist",
+            name="updated_date",
+            field=models.DateTimeField(auto_now=True),
+        ),
+    ]

bookwyrm/migrations/0159_auto_20220924_0634.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.15 on 2022-09-24 06:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0158_auto_20220919_1634"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="hotp_count",
+            field=models.IntegerField(blank=True, default=0, null=True),
+        ),
+        migrations.AddField(
+            model_name="user",
+            name="hotp_secret",
+            field=models.CharField(blank=True, default=None, max_length=32, null=True),
+        ),
+        migrations.AddField(
+            model_name="user",
+            name="otp_secret",
+            field=models.CharField(blank=True, default=None, max_length=32, null=True),
+        ),
+        migrations.AddField(
+            model_name="user",
+            name="two_factor_auth",
+            field=models.BooleanField(blank=True, default=None, null=True),
+        ),
+    ]

bookwyrm/migrations/0160_auto_20221101_2251.py

@@ -0,0 +1,52 @@
+# Generated by Django 3.2.15 on 2022-11-01 22:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0159_auto_20220924_0634"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="allow_reactivation",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name="connector",
+            name="deactivation_reason",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("pending", "Pending"),
+                    ("self_deletion", "Self deletion"),
+                    ("self_deactivation", "Self deactivation"),
+                    ("moderator_suspension", "Moderator suspension"),
+                    ("moderator_deletion", "Moderator deletion"),
+                    ("domain_block", "Domain block"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="user",
+            name="deactivation_reason",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("pending", "Pending"),
+                    ("self_deletion", "Self deletion"),
+                    ("self_deactivation", "Self deactivation"),
+                    ("moderator_suspension", "Moderator suspension"),
+                    ("moderator_deletion", "Moderator deletion"),
+                    ("domain_block", "Domain block"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0160_auto_20221105_2030.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.15 on 2022-11-05 20:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0159_auto_20220924_0634"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importitem",
+            name="task_id",
+            field=models.CharField(blank=True, max_length=200, null=True),
+        ),
+        migrations.AddField(
+            model_name="importjob",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("pending", "Pending"),
+                    ("active", "Active"),
+                    ("complete", "Complete"),
+                    ("stopped", "Stopped"),
+                ],
+                max_length=50,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0161_alter_importjob_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.15 on 2022-11-05 20:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0160_auto_20221105_2030"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="importjob",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("pending", "Pending"),
+                    ("active", "Active"),
+                    ("complete", "Complete"),
+                    ("stopped", "Stopped"),
+                ],
+                default="pending",
+                max_length=50,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0162_importjob_task_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2022-11-05 22:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0161_alter_importjob_status"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="importjob",
+            name="task_id",
+            field=models.CharField(blank=True, max_length=200, null=True),
+        ),
+    ]

bookwyrm/migrations/0163_merge_0160_auto_20221101_2251_0162_importjob_task_id.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.15 on 2022-11-10 20:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0160_auto_20221101_2251"),
+        ("bookwyrm", "0162_importjob_task_id"),
+    ]
+
+    operations = []

bookwyrm/migrations/0164_status_ready.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-15 21:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0163_merge_0160_auto_20221101_2251_0162_importjob_task_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="status",
+            name="ready",
+            field=models.BooleanField(default=True),
+        ),
+    ]

bookwyrm/migrations/0165_alter_inviterequest_answer.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-15 22:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0164_status_ready"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="inviterequest",
+            name="answer",
+            field=models.TextField(blank=True, max_length=255, null=True),
+        ),
+    ]

bookwyrm/migrations/0166_sitesettings_imports_enabled.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-17 21:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0165_alter_inviterequest_answer"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="imports_enabled",
+            field=models.BooleanField(default=True),
+        ),
+    ]

bookwyrm/migrations/0167_auto_20221125_1900.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-11-25 19:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0166_sitesettings_imports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="impressum",
+            field=models.TextField(default="Add a impressum here."),
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="show_impressum",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/migrations/0167_sitesettings_import_size_limit.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2022-12-05 13:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0166_sitesettings_imports_enabled"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="import_size_limit",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="sitesettings",
+            name="import_limit_reset",
+            field=models.IntegerField(default=0),
+        ),
+    ]

bookwyrm/migrations/0168_auto_20221205_1701.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.16 on 2022-12-05 17:01
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0167_auto_20221125_1900"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="aasin",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="aasin",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0168_auto_20221205_2331.py

@@ -0,0 +1,63 @@
+""" I added two new permission types and a new group to the management command that
+creates the database on install, this creates them for existing instances """
+# Generated by Django 3.2.16 on 2022-12-05 23:31
+
+from django.db import migrations
+
+
+def create_groups_and_perms(apps, schema_editor):
+    """create the new "owner" group and "system admin" permission"""
+    db_alias = schema_editor.connection.alias
+    group_model = apps.get_model("auth", "Group")
+    # Add the "owner" group, if needed
+    owner_group, group_created = group_model.objects.using(db_alias).get_or_create(
+        name="owner"
+    )
+
+    # Create perms, if needed
+    user_model = apps.get_model("bookwyrm", "User")
+    content_type_model = apps.get_model("contenttypes", "ContentType")
+    content_type = content_type_model.objects.get_for_model(user_model)
+    perms_model = apps.get_model("auth", "Permission")
+    reg_perm, perm_created = perms_model.objects.using(db_alias).get_or_create(
+        codename="manage_registration",
+        name="allow or prevent user registration",
+        content_type=content_type,
+    )
+    admin_perm, admin_perm_created = perms_model.objects.using(db_alias).get_or_create(
+        codename="system_administration",
+        name="technical controls",
+        content_type=content_type,
+    )
+
+    # Add perms to the group if anything was created
+    if group_created or perm_created or admin_perm_created:
+        perms = [
+            "edit_instance_settings",
+            "set_user_group",
+            "control_federation",
+            "create_invites",
+            "moderate_user",
+            "moderate_post",
+            "edit_book",
+        ]
+        owner_group.permissions.set(
+            perms_model.objects.using(db_alias).filter(codename__in=perms).all()
+        )
+
+        # also extend these perms to admins
+        # This is get or create so the tests don't fail -- it should already exist
+        admin_group, _ = group_model.objects.using(db_alias).get_or_create(name="admin")
+        admin_group.permissions.add(reg_perm)
+        admin_group.permissions.add(admin_perm)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0167_auto_20221125_1900"),
+    ]
+
+    operations = [
+        migrations.RunPython(create_groups_and_perms, migrations.RunPython.noop)
+    ]

bookwyrm/migrations/0169_auto_20221206_0902.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.16 on 2022-12-06 09:02
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0168_auto_20221205_1701"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="isfdb",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="isfdb",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2022-12-11 20:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0168_auto_20221205_2331"),
+        ("bookwyrm", "0169_auto_20221206_0902"),
+    ]
+
+    operations = []

bookwyrm/migrations/0171_alter_user_preferred_timezone.py

@@ -0,0 +1,631 @@
+# Generated by Django 3.2.16 on 2022-12-19 15:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_timezone",
+            field=models.CharField(
+                choices=[
+                    ("Africa/Abidjan", "Africa/Abidjan"),
+                    ("Africa/Accra", "Africa/Accra"),
+                    ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+                    ("Africa/Algiers", "Africa/Algiers"),
+                    ("Africa/Asmara", "Africa/Asmara"),
+                    ("Africa/Asmera", "Africa/Asmera"),
+                    ("Africa/Bamako", "Africa/Bamako"),
+                    ("Africa/Bangui", "Africa/Bangui"),
+                    ("Africa/Banjul", "Africa/Banjul"),
+                    ("Africa/Bissau", "Africa/Bissau"),
+                    ("Africa/Blantyre", "Africa/Blantyre"),
+                    ("Africa/Brazzaville", "Africa/Brazzaville"),
+                    ("Africa/Bujumbura", "Africa/Bujumbura"),
+                    ("Africa/Cairo", "Africa/Cairo"),
+                    ("Africa/Casablanca", "Africa/Casablanca"),
+                    ("Africa/Ceuta", "Africa/Ceuta"),
+                    ("Africa/Conakry", "Africa/Conakry"),
+                    ("Africa/Dakar", "Africa/Dakar"),
+                    ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+                    ("Africa/Djibouti", "Africa/Djibouti"),
+                    ("Africa/Douala", "Africa/Douala"),
+                    ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+                    ("Africa/Freetown", "Africa/Freetown"),
+                    ("Africa/Gaborone", "Africa/Gaborone"),
+                    ("Africa/Harare", "Africa/Harare"),
+                    ("Africa/Johannesburg", "Africa/Johannesburg"),
+                    ("Africa/Juba", "Africa/Juba"),
+                    ("Africa/Kampala", "Africa/Kampala"),
+                    ("Africa/Khartoum", "Africa/Khartoum"),
+                    ("Africa/Kigali", "Africa/Kigali"),
+                    ("Africa/Kinshasa", "Africa/Kinshasa"),
+                    ("Africa/Lagos", "Africa/Lagos"),
+                    ("Africa/Libreville", "Africa/Libreville"),
+                    ("Africa/Lome", "Africa/Lome"),
+                    ("Africa/Luanda", "Africa/Luanda"),
+                    ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "Africa/Lusaka"),
+                    ("Africa/Malabo", "Africa/Malabo"),
+                    ("Africa/Maputo", "Africa/Maputo"),
+                    ("Africa/Maseru", "Africa/Maseru"),
+                    ("Africa/Mbabane", "Africa/Mbabane"),
+                    ("Africa/Mogadishu", "Africa/Mogadishu"),
+                    ("Africa/Monrovia", "Africa/Monrovia"),
+                    ("Africa/Nairobi", "Africa/Nairobi"),
+                    ("Africa/Ndjamena", "Africa/Ndjamena"),
+                    ("Africa/Niamey", "Africa/Niamey"),
+                    ("Africa/Nouakchott", "Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+                    ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+                    ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+                    ("Africa/Timbuktu", "Africa/Timbuktu"),
+                    ("Africa/Tripoli", "Africa/Tripoli"),
+                    ("Africa/Tunis", "Africa/Tunis"),
+                    ("Africa/Windhoek", "Africa/Windhoek"),
+                    ("America/Adak", "America/Adak"),
+                    ("America/Anchorage", "America/Anchorage"),
+                    ("America/Anguilla", "America/Anguilla"),
+                    ("America/Antigua", "America/Antigua"),
+                    ("America/Araguaina", "America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "America/Argentina/Buenos_Aires",
+                    ),
+                    ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+                    (
+                        "America/Argentina/ComodRivadavia",
+                        "America/Argentina/ComodRivadavia",
+                    ),
+                    ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+                    ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+                    ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+                    ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "America/Argentina/Rio_Gallegos",
+                    ),
+                    ("America/Argentina/Salta", "America/Argentina/Salta"),
+                    ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+                    ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+                    ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+                    ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+                    ("America/Aruba", "America/Aruba"),
+                    ("America/Asuncion", "America/Asuncion"),
+                    ("America/Atikokan", "America/Atikokan"),
+                    ("America/Atka", "America/Atka"),
+                    ("America/Bahia", "America/Bahia"),
+                    ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+                    ("America/Barbados", "America/Barbados"),
+                    ("America/Belem", "America/Belem"),
+                    ("America/Belize", "America/Belize"),
+                    ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+                    ("America/Boa_Vista", "America/Boa_Vista"),
+                    ("America/Bogota", "America/Bogota"),
+                    ("America/Boise", "America/Boise"),
+                    ("America/Buenos_Aires", "America/Buenos_Aires"),
+                    ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+                    ("America/Campo_Grande", "America/Campo_Grande"),
+                    ("America/Cancun", "America/Cancun"),
+                    ("America/Caracas", "America/Caracas"),
+                    ("America/Catamarca", "America/Catamarca"),
+                    ("America/Cayenne", "America/Cayenne"),
+                    ("America/Cayman", "America/Cayman"),
+                    ("America/Chicago", "America/Chicago"),
+                    ("America/Chihuahua", "America/Chihuahua"),
+                    ("America/Ciudad_Juarez", "America/Ciudad_Juarez"),
+                    ("America/Coral_Harbour", "America/Coral_Harbour"),
+                    ("America/Cordoba", "America/Cordoba"),
+                    ("America/Costa_Rica", "America/Costa_Rica"),
+                    ("America/Creston", "America/Creston"),
+                    ("America/Cuiaba", "America/Cuiaba"),
+                    ("America/Curacao", "America/Curacao"),
+                    ("America/Danmarkshavn", "America/Danmarkshavn"),
+                    ("America/Dawson", "America/Dawson"),
+                    ("America/Dawson_Creek", "America/Dawson_Creek"),
+                    ("America/Denver", "America/Denver"),
+                    ("America/Detroit", "America/Detroit"),
+                    ("America/Dominica", "America/Dominica"),
+                    ("America/Edmonton", "America/Edmonton"),
+                    ("America/Eirunepe", "America/Eirunepe"),
+                    ("America/El_Salvador", "America/El_Salvador"),
+                    ("America/Ensenada", "America/Ensenada"),
+                    ("America/Fort_Nelson", "America/Fort_Nelson"),
+                    ("America/Fort_Wayne", "America/Fort_Wayne"),
+                    ("America/Fortaleza", "America/Fortaleza"),
+                    ("America/Glace_Bay", "America/Glace_Bay"),
+                    ("America/Godthab", "America/Godthab"),
+                    ("America/Goose_Bay", "America/Goose_Bay"),
+                    ("America/Grand_Turk", "America/Grand_Turk"),
+                    ("America/Grenada", "America/Grenada"),
+                    ("America/Guadeloupe", "America/Guadeloupe"),
+                    ("America/Guatemala", "America/Guatemala"),
+                    ("America/Guayaquil", "America/Guayaquil"),
+                    ("America/Guyana", "America/Guyana"),
+                    ("America/Halifax", "America/Halifax"),
+                    ("America/Havana", "America/Havana"),
+                    ("America/Hermosillo", "America/Hermosillo"),
+                    ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+                    ("America/Indiana/Knox", "America/Indiana/Knox"),
+                    ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+                    ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+                    ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+                    ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+                    ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+                    ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+                    ("America/Indianapolis", "America/Indianapolis"),
+                    ("America/Inuvik", "America/Inuvik"),
+                    ("America/Iqaluit", "America/Iqaluit"),
+                    ("America/Jamaica", "America/Jamaica"),
+                    ("America/Jujuy", "America/Jujuy"),
+                    ("America/Juneau", "America/Juneau"),
+                    ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+                    ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+                    ("America/Knox_IN", "America/Knox_IN"),
+                    ("America/Kralendijk", "America/Kralendijk"),
+                    ("America/La_Paz", "America/La_Paz"),
+                    ("America/Lima", "America/Lima"),
+                    ("America/Los_Angeles", "America/Los_Angeles"),
+                    ("America/Louisville", "America/Louisville"),
+                    ("America/Lower_Princes", "America/Lower_Princes"),
+                    ("America/Maceio", "America/Maceio"),
+                    ("America/Managua", "America/Managua"),
+                    ("America/Manaus", "America/Manaus"),
+                    ("America/Marigot", "America/Marigot"),
+                    ("America/Martinique", "America/Martinique"),
+                    ("America/Matamoros", "America/Matamoros"),
+                    ("America/Mazatlan", "America/Mazatlan"),
+                    ("America/Mendoza", "America/Mendoza"),
+                    ("America/Menominee", "America/Menominee"),
+                    ("America/Merida", "America/Merida"),
+                    ("America/Metlakatla", "America/Metlakatla"),
+                    ("America/Mexico_City", "America/Mexico_City"),
+                    ("America/Miquelon", "America/Miquelon"),
+                    ("America/Moncton", "America/Moncton"),
+                    ("America/Monterrey", "America/Monterrey"),
+                    ("America/Montevideo", "America/Montevideo"),
+                    ("America/Montreal", "America/Montreal"),
+                    ("America/Montserrat", "America/Montserrat"),
+                    ("America/Nassau", "America/Nassau"),
+                    ("America/New_York", "America/New_York"),
+                    ("America/Nipigon", "America/Nipigon"),
+                    ("America/Nome", "America/Nome"),
+                    ("America/Noronha", "America/Noronha"),
+                    ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+                    ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Nuuk", "America/Nuuk"),
+                    ("America/Ojinaga", "America/Ojinaga"),
+                    ("America/Panama", "America/Panama"),
+                    ("America/Pangnirtung", "America/Pangnirtung"),
+                    ("America/Paramaribo", "America/Paramaribo"),
+                    ("America/Phoenix", "America/Phoenix"),
+                    ("America/Port-au-Prince", "America/Port-au-Prince"),
+                    ("America/Port_of_Spain", "America/Port_of_Spain"),
+                    ("America/Porto_Acre", "America/Porto_Acre"),
+                    ("America/Porto_Velho", "America/Porto_Velho"),
+                    ("America/Puerto_Rico", "America/Puerto_Rico"),
+                    ("America/Punta_Arenas", "America/Punta_Arenas"),
+                    ("America/Rainy_River", "America/Rainy_River"),
+                    ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+                    ("America/Recife", "America/Recife"),
+                    ("America/Regina", "America/Regina"),
+                    ("America/Resolute", "America/Resolute"),
+                    ("America/Rio_Branco", "America/Rio_Branco"),
+                    ("America/Rosario", "America/Rosario"),
+                    ("America/Santa_Isabel", "America/Santa_Isabel"),
+                    ("America/Santarem", "America/Santarem"),
+                    ("America/Santiago", "America/Santiago"),
+                    ("America/Santo_Domingo", "America/Santo_Domingo"),
+                    ("America/Sao_Paulo", "America/Sao_Paulo"),
+                    ("America/Scoresbysund", "America/Scoresbysund"),
+                    ("America/Shiprock", "America/Shiprock"),
+                    ("America/Sitka", "America/Sitka"),
+                    ("America/St_Barthelemy", "America/St_Barthelemy"),
+                    ("America/St_Johns", "America/St_Johns"),
+                    ("America/St_Kitts", "America/St_Kitts"),
+                    ("America/St_Lucia", "America/St_Lucia"),
+                    ("America/St_Thomas", "America/St_Thomas"),
+                    ("America/St_Vincent", "America/St_Vincent"),
+                    ("America/Swift_Current", "America/Swift_Current"),
+                    ("America/Tegucigalpa", "America/Tegucigalpa"),
+                    ("America/Thule", "America/Thule"),
+                    ("America/Thunder_Bay", "America/Thunder_Bay"),
+                    ("America/Tijuana", "America/Tijuana"),
+                    ("America/Toronto", "America/Toronto"),
+                    ("America/Tortola", "America/Tortola"),
+                    ("America/Vancouver", "America/Vancouver"),
+                    ("America/Virgin", "America/Virgin"),
+                    ("America/Whitehorse", "America/Whitehorse"),
+                    ("America/Winnipeg", "America/Winnipeg"),
+                    ("America/Yakutat", "America/Yakutat"),
+                    ("America/Yellowknife", "America/Yellowknife"),
+                    ("Antarctica/Casey", "Antarctica/Casey"),
+                    ("Antarctica/Davis", "Antarctica/Davis"),
+                    ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+                    ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+                    ("Antarctica/Mawson", "Antarctica/Mawson"),
+                    ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+                    ("Antarctica/Palmer", "Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "Antarctica/Rothera"),
+                    ("Antarctica/South_Pole", "Antarctica/South_Pole"),
+                    ("Antarctica/Syowa", "Antarctica/Syowa"),
+                    ("Antarctica/Troll", "Antarctica/Troll"),
+                    ("Antarctica/Vostok", "Antarctica/Vostok"),
+                    ("Arctic/Longyearbyen", "Arctic/Longyearbyen"),
+                    ("Asia/Aden", "Asia/Aden"),
+                    ("Asia/Almaty", "Asia/Almaty"),
+                    ("Asia/Amman", "Asia/Amman"),
+                    ("Asia/Anadyr", "Asia/Anadyr"),
+                    ("Asia/Aqtau", "Asia/Aqtau"),
+                    ("Asia/Aqtobe", "Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "Asia/Ashgabat"),
+                    ("Asia/Ashkhabad", "Asia/Ashkhabad"),
+                    ("Asia/Atyrau", "Asia/Atyrau"),
+                    ("Asia/Baghdad", "Asia/Baghdad"),
+                    ("Asia/Bahrain", "Asia/Bahrain"),
+                    ("Asia/Baku", "Asia/Baku"),
+                    ("Asia/Bangkok", "Asia/Bangkok"),
+                    ("Asia/Barnaul", "Asia/Barnaul"),
+                    ("Asia/Beirut", "Asia/Beirut"),
+                    ("Asia/Bishkek", "Asia/Bishkek"),
+                    ("Asia/Brunei", "Asia/Brunei"),
+                    ("Asia/Calcutta", "Asia/Calcutta"),
+                    ("Asia/Chita", "Asia/Chita"),
+                    ("Asia/Choibalsan", "Asia/Choibalsan"),
+                    ("Asia/Chongqing", "Asia/Chongqing"),
+                    ("Asia/Chungking", "Asia/Chungking"),
+                    ("Asia/Colombo", "Asia/Colombo"),
+                    ("Asia/Dacca", "Asia/Dacca"),
+                    ("Asia/Damascus", "Asia/Damascus"),
+                    ("Asia/Dhaka", "Asia/Dhaka"),
+                    ("Asia/Dili", "Asia/Dili"),
+                    ("Asia/Dubai", "Asia/Dubai"),
+                    ("Asia/Dushanbe", "Asia/Dushanbe"),
+                    ("Asia/Famagusta", "Asia/Famagusta"),
+                    ("Asia/Gaza", "Asia/Gaza"),
+                    ("Asia/Harbin", "Asia/Harbin"),
+                    ("Asia/Hebron", "Asia/Hebron"),
+                    ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+                    ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+                    ("Asia/Hovd", "Asia/Hovd"),
+                    ("Asia/Irkutsk", "Asia/Irkutsk"),
+                    ("Asia/Istanbul", "Asia/Istanbul"),
+                    ("Asia/Jakarta", "Asia/Jakarta"),
+                    ("Asia/Jayapura", "Asia/Jayapura"),
+                    ("Asia/Jerusalem", "Asia/Jerusalem"),
+                    ("Asia/Kabul", "Asia/Kabul"),
+                    ("Asia/Kamchatka", "Asia/Kamchatka"),
+                    ("Asia/Karachi", "Asia/Karachi"),
+                    ("Asia/Kashgar", "Asia/Kashgar"),
+                    ("Asia/Kathmandu", "Asia/Kathmandu"),
+                    ("Asia/Katmandu", "Asia/Katmandu"),
+                    ("Asia/Khandyga", "Asia/Khandyga"),
+                    ("Asia/Kolkata", "Asia/Kolkata"),
+                    ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+                    ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "Asia/Kuching"),
+                    ("Asia/Kuwait", "Asia/Kuwait"),
+                    ("Asia/Macao", "Asia/Macao"),
+                    ("Asia/Macau", "Asia/Macau"),
+                    ("Asia/Magadan", "Asia/Magadan"),
+                    ("Asia/Makassar", "Asia/Makassar"),
+                    ("Asia/Manila", "Asia/Manila"),
+                    ("Asia/Muscat", "Asia/Muscat"),
+                    ("Asia/Nicosia", "Asia/Nicosia"),
+                    ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+                    ("Asia/Omsk", "Asia/Omsk"),
+                    ("Asia/Oral", "Asia/Oral"),
+                    ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "Asia/Pontianak"),
+                    ("Asia/Pyongyang", "Asia/Pyongyang"),
+                    ("Asia/Qatar", "Asia/Qatar"),
+                    ("Asia/Qostanay", "Asia/Qostanay"),
+                    ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+                    ("Asia/Rangoon", "Asia/Rangoon"),
+                    ("Asia/Riyadh", "Asia/Riyadh"),
+                    ("Asia/Saigon", "Asia/Saigon"),
+                    ("Asia/Sakhalin", "Asia/Sakhalin"),
+                    ("Asia/Samarkand", "Asia/Samarkand"),
+                    ("Asia/Seoul", "Asia/Seoul"),
+                    ("Asia/Shanghai", "Asia/Shanghai"),
+                    ("Asia/Singapore", "Asia/Singapore"),
+                    ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+                    ("Asia/Taipei", "Asia/Taipei"),
+                    ("Asia/Tashkent", "Asia/Tashkent"),
+                    ("Asia/Tbilisi", "Asia/Tbilisi"),
+                    ("Asia/Tehran", "Asia/Tehran"),
+                    ("Asia/Tel_Aviv", "Asia/Tel_Aviv"),
+                    ("Asia/Thimbu", "Asia/Thimbu"),
+                    ("Asia/Thimphu", "Asia/Thimphu"),
+                    ("Asia/Tokyo", "Asia/Tokyo"),
+                    ("Asia/Tomsk", "Asia/Tomsk"),
+                    ("Asia/Ujung_Pandang", "Asia/Ujung_Pandang"),
+                    ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+                    ("Asia/Ulan_Bator", "Asia/Ulan_Bator"),
+                    ("Asia/Urumqi", "Asia/Urumqi"),
+                    ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+                    ("Asia/Vientiane", "Asia/Vientiane"),
+                    ("Asia/Vladivostok", "Asia/Vladivostok"),
+                    ("Asia/Yakutsk", "Asia/Yakutsk"),
+                    ("Asia/Yangon", "Asia/Yangon"),
+                    ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+                    ("Asia/Yerevan", "Asia/Yerevan"),
+                    ("Atlantic/Azores", "Atlantic/Azores"),
+                    ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+                    ("Atlantic/Canary", "Atlantic/Canary"),
+                    ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+                    ("Atlantic/Faeroe", "Atlantic/Faeroe"),
+                    ("Atlantic/Faroe", "Atlantic/Faroe"),
+                    ("Atlantic/Jan_Mayen", "Atlantic/Jan_Mayen"),
+                    ("Atlantic/Madeira", "Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+                    ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+                    ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+                    ("Atlantic/Stanley", "Atlantic/Stanley"),
+                    ("Australia/ACT", "Australia/ACT"),
+                    ("Australia/Adelaide", "Australia/Adelaide"),
+                    ("Australia/Brisbane", "Australia/Brisbane"),
+                    ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+                    ("Australia/Canberra", "Australia/Canberra"),
+                    ("Australia/Currie", "Australia/Currie"),
+                    ("Australia/Darwin", "Australia/Darwin"),
+                    ("Australia/Eucla", "Australia/Eucla"),
+                    ("Australia/Hobart", "Australia/Hobart"),
+                    ("Australia/LHI", "Australia/LHI"),
+                    ("Australia/Lindeman", "Australia/Lindeman"),
+                    ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "Australia/Melbourne"),
+                    ("Australia/NSW", "Australia/NSW"),
+                    ("Australia/North", "Australia/North"),
+                    ("Australia/Perth", "Australia/Perth"),
+                    ("Australia/Queensland", "Australia/Queensland"),
+                    ("Australia/South", "Australia/South"),
+                    ("Australia/Sydney", "Australia/Sydney"),
+                    ("Australia/Tasmania", "Australia/Tasmania"),
+                    ("Australia/Victoria", "Australia/Victoria"),
+                    ("Australia/West", "Australia/West"),
+                    ("Australia/Yancowinna", "Australia/Yancowinna"),
+                    ("Brazil/Acre", "Brazil/Acre"),
+                    ("Brazil/DeNoronha", "Brazil/DeNoronha"),
+                    ("Brazil/East", "Brazil/East"),
+                    ("Brazil/West", "Brazil/West"),
+                    ("CET", "CET"),
+                    ("CST6CDT", "CST6CDT"),
+                    ("Canada/Atlantic", "Canada/Atlantic"),
+                    ("Canada/Central", "Canada/Central"),
+                    ("Canada/Eastern", "Canada/Eastern"),
+                    ("Canada/Mountain", "Canada/Mountain"),
+                    ("Canada/Newfoundland", "Canada/Newfoundland"),
+                    ("Canada/Pacific", "Canada/Pacific"),
+                    ("Canada/Saskatchewan", "Canada/Saskatchewan"),
+                    ("Canada/Yukon", "Canada/Yukon"),
+                    ("Chile/Continental", "Chile/Continental"),
+                    ("Chile/EasterIsland", "Chile/EasterIsland"),
+                    ("Cuba", "Cuba"),
+                    ("EET", "EET"),
+                    ("EST", "EST"),
+                    ("EST5EDT", "EST5EDT"),
+                    ("Egypt", "Egypt"),
+                    ("Eire", "Eire"),
+                    ("Etc/GMT", "Etc/GMT"),
+                    ("Etc/GMT+0", "Etc/GMT+0"),
+                    ("Etc/GMT+1", "Etc/GMT+1"),
+                    ("Etc/GMT+10", "Etc/GMT+10"),
+                    ("Etc/GMT+11", "Etc/GMT+11"),
+                    ("Etc/GMT+12", "Etc/GMT+12"),
+                    ("Etc/GMT+2", "Etc/GMT+2"),
+                    ("Etc/GMT+3", "Etc/GMT+3"),
+                    ("Etc/GMT+4", "Etc/GMT+4"),
+                    ("Etc/GMT+5", "Etc/GMT+5"),
+                    ("Etc/GMT+6", "Etc/GMT+6"),
+                    ("Etc/GMT+7", "Etc/GMT+7"),
+                    ("Etc/GMT+8", "Etc/GMT+8"),
+                    ("Etc/GMT+9", "Etc/GMT+9"),
+                    ("Etc/GMT-0", "Etc/GMT-0"),
+                    ("Etc/GMT-1", "Etc/GMT-1"),
+                    ("Etc/GMT-10", "Etc/GMT-10"),
+                    ("Etc/GMT-11", "Etc/GMT-11"),
+                    ("Etc/GMT-12", "Etc/GMT-12"),
+                    ("Etc/GMT-13", "Etc/GMT-13"),
+                    ("Etc/GMT-14", "Etc/GMT-14"),
+                    ("Etc/GMT-2", "Etc/GMT-2"),
+                    ("Etc/GMT-3", "Etc/GMT-3"),
+                    ("Etc/GMT-4", "Etc/GMT-4"),
+                    ("Etc/GMT-5", "Etc/GMT-5"),
+                    ("Etc/GMT-6", "Etc/GMT-6"),
+                    ("Etc/GMT-7", "Etc/GMT-7"),
+                    ("Etc/GMT-8", "Etc/GMT-8"),
+                    ("Etc/GMT-9", "Etc/GMT-9"),
+                    ("Etc/GMT0", "Etc/GMT0"),
+                    ("Etc/Greenwich", "Etc/Greenwich"),
+                    ("Etc/UCT", "Etc/UCT"),
+                    ("Etc/UTC", "Etc/UTC"),
+                    ("Etc/Universal", "Etc/Universal"),
+                    ("Etc/Zulu", "Etc/Zulu"),
+                    ("Europe/Amsterdam", "Europe/Amsterdam"),
+                    ("Europe/Andorra", "Europe/Andorra"),
+                    ("Europe/Astrakhan", "Europe/Astrakhan"),
+                    ("Europe/Athens", "Europe/Athens"),
+                    ("Europe/Belfast", "Europe/Belfast"),
+                    ("Europe/Belgrade", "Europe/Belgrade"),
+                    ("Europe/Berlin", "Europe/Berlin"),
+                    ("Europe/Bratislava", "Europe/Bratislava"),
+                    ("Europe/Brussels", "Europe/Brussels"),
+                    ("Europe/Bucharest", "Europe/Bucharest"),
+                    ("Europe/Budapest", "Europe/Budapest"),
+                    ("Europe/Busingen", "Europe/Busingen"),
+                    ("Europe/Chisinau", "Europe/Chisinau"),
+                    ("Europe/Copenhagen", "Europe/Copenhagen"),
+                    ("Europe/Dublin", "Europe/Dublin"),
+                    ("Europe/Gibraltar", "Europe/Gibraltar"),
+                    ("Europe/Guernsey", "Europe/Guernsey"),
+                    ("Europe/Helsinki", "Europe/Helsinki"),
+                    ("Europe/Isle_of_Man", "Europe/Isle_of_Man"),
+                    ("Europe/Istanbul", "Europe/Istanbul"),
+                    ("Europe/Jersey", "Europe/Jersey"),
+                    ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+                    ("Europe/Kiev", "Europe/Kiev"),
+                    ("Europe/Kirov", "Europe/Kirov"),
+                    ("Europe/Kyiv", "Europe/Kyiv"),
+                    ("Europe/Lisbon", "Europe/Lisbon"),
+                    ("Europe/Ljubljana", "Europe/Ljubljana"),
+                    ("Europe/London", "Europe/London"),
+                    ("Europe/Luxembourg", "Europe/Luxembourg"),
+                    ("Europe/Madrid", "Europe/Madrid"),
+                    ("Europe/Malta", "Europe/Malta"),
+                    ("Europe/Mariehamn", "Europe/Mariehamn"),
+                    ("Europe/Minsk", "Europe/Minsk"),
+                    ("Europe/Monaco", "Europe/Monaco"),
+                    ("Europe/Moscow", "Europe/Moscow"),
+                    ("Europe/Nicosia", "Europe/Nicosia"),
+                    ("Europe/Oslo", "Europe/Oslo"),
+                    ("Europe/Paris", "Europe/Paris"),
+                    ("Europe/Podgorica", "Europe/Podgorica"),
+                    ("Europe/Prague", "Europe/Prague"),
+                    ("Europe/Riga", "Europe/Riga"),
+                    ("Europe/Rome", "Europe/Rome"),
+                    ("Europe/Samara", "Europe/Samara"),
+                    ("Europe/San_Marino", "Europe/San_Marino"),
+                    ("Europe/Sarajevo", "Europe/Sarajevo"),
+                    ("Europe/Saratov", "Europe/Saratov"),
+                    ("Europe/Simferopol", "Europe/Simferopol"),
+                    ("Europe/Skopje", "Europe/Skopje"),
+                    ("Europe/Sofia", "Europe/Sofia"),
+                    ("Europe/Stockholm", "Europe/Stockholm"),
+                    ("Europe/Tallinn", "Europe/Tallinn"),
+                    ("Europe/Tirane", "Europe/Tirane"),
+                    ("Europe/Tiraspol", "Europe/Tiraspol"),
+                    ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+                    ("Europe/Uzhgorod", "Europe/Uzhgorod"),
+                    ("Europe/Vaduz", "Europe/Vaduz"),
+                    ("Europe/Vatican", "Europe/Vatican"),
+                    ("Europe/Vienna", "Europe/Vienna"),
+                    ("Europe/Vilnius", "Europe/Vilnius"),
+                    ("Europe/Volgograd", "Europe/Volgograd"),
+                    ("Europe/Warsaw", "Europe/Warsaw"),
+                    ("Europe/Zagreb", "Europe/Zagreb"),
+                    ("Europe/Zaporozhye", "Europe/Zaporozhye"),
+                    ("Europe/Zurich", "Europe/Zurich"),
+                    ("GB", "GB"),
+                    ("GB-Eire", "GB-Eire"),
+                    ("GMT", "GMT"),
+                    ("GMT+0", "GMT+0"),
+                    ("GMT-0", "GMT-0"),
+                    ("GMT0", "GMT0"),
+                    ("Greenwich", "Greenwich"),
+                    ("HST", "HST"),
+                    ("Hongkong", "Hongkong"),
+                    ("Iceland", "Iceland"),
+                    ("Indian/Antananarivo", "Indian/Antananarivo"),
+                    ("Indian/Chagos", "Indian/Chagos"),
+                    ("Indian/Christmas", "Indian/Christmas"),
+                    ("Indian/Cocos", "Indian/Cocos"),
+                    ("Indian/Comoro", "Indian/Comoro"),
+                    ("Indian/Kerguelen", "Indian/Kerguelen"),
+                    ("Indian/Mahe", "Indian/Mahe"),
+                    ("Indian/Maldives", "Indian/Maldives"),
+                    ("Indian/Mauritius", "Indian/Mauritius"),
+                    ("Indian/Mayotte", "Indian/Mayotte"),
+                    ("Indian/Reunion", "Indian/Reunion"),
+                    ("Iran", "Iran"),
+                    ("Israel", "Israel"),
+                    ("Jamaica", "Jamaica"),
+                    ("Japan", "Japan"),
+                    ("Kwajalein", "Kwajalein"),
+                    ("Libya", "Libya"),
+                    ("MET", "MET"),
+                    ("MST", "MST"),
+                    ("MST7MDT", "MST7MDT"),
+                    ("Mexico/BajaNorte", "Mexico/BajaNorte"),
+                    ("Mexico/BajaSur", "Mexico/BajaSur"),
+                    ("Mexico/General", "Mexico/General"),
+                    ("NZ", "NZ"),
+                    ("NZ-CHAT", "NZ-CHAT"),
+                    ("Navajo", "Navajo"),
+                    ("PRC", "PRC"),
+                    ("PST8PDT", "PST8PDT"),
+                    ("Pacific/Apia", "Pacific/Apia"),
+                    ("Pacific/Auckland", "Pacific/Auckland"),
+                    ("Pacific/Bougainville", "Pacific/Bougainville"),
+                    ("Pacific/Chatham", "Pacific/Chatham"),
+                    ("Pacific/Chuuk", "Pacific/Chuuk"),
+                    ("Pacific/Easter", "Pacific/Easter"),
+                    ("Pacific/Efate", "Pacific/Efate"),
+                    ("Pacific/Enderbury", "Pacific/Enderbury"),
+                    ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+                    ("Pacific/Fiji", "Pacific/Fiji"),
+                    ("Pacific/Funafuti", "Pacific/Funafuti"),
+                    ("Pacific/Galapagos", "Pacific/Galapagos"),
+                    ("Pacific/Gambier", "Pacific/Gambier"),
+                    ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+                    ("Pacific/Guam", "Pacific/Guam"),
+                    ("Pacific/Honolulu", "Pacific/Honolulu"),
+                    ("Pacific/Johnston", "Pacific/Johnston"),
+                    ("Pacific/Kanton", "Pacific/Kanton"),
+                    ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+                    ("Pacific/Kosrae", "Pacific/Kosrae"),
+                    ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "Pacific/Majuro"),
+                    ("Pacific/Marquesas", "Pacific/Marquesas"),
+                    ("Pacific/Midway", "Pacific/Midway"),
+                    ("Pacific/Nauru", "Pacific/Nauru"),
+                    ("Pacific/Niue", "Pacific/Niue"),
+                    ("Pacific/Norfolk", "Pacific/Norfolk"),
+                    ("Pacific/Noumea", "Pacific/Noumea"),
+                    ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+                    ("Pacific/Palau", "Pacific/Palau"),
+                    ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+                    ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+                    ("Pacific/Ponape", "Pacific/Ponape"),
+                    ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+                    ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+                    ("Pacific/Saipan", "Pacific/Saipan"),
+                    ("Pacific/Samoa", "Pacific/Samoa"),
+                    ("Pacific/Tahiti", "Pacific/Tahiti"),
+                    ("Pacific/Tarawa", "Pacific/Tarawa"),
+                    ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+                    ("Pacific/Truk", "Pacific/Truk"),
+                    ("Pacific/Wake", "Pacific/Wake"),
+                    ("Pacific/Wallis", "Pacific/Wallis"),
+                    ("Pacific/Yap", "Pacific/Yap"),
+                    ("Poland", "Poland"),
+                    ("Portugal", "Portugal"),
+                    ("ROC", "ROC"),
+                    ("ROK", "ROK"),
+                    ("Singapore", "Singapore"),
+                    ("Turkey", "Turkey"),
+                    ("UCT", "UCT"),
+                    ("US/Alaska", "US/Alaska"),
+                    ("US/Aleutian", "US/Aleutian"),
+                    ("US/Arizona", "US/Arizona"),
+                    ("US/Central", "US/Central"),
+                    ("US/East-Indiana", "US/East-Indiana"),
+                    ("US/Eastern", "US/Eastern"),
+                    ("US/Hawaii", "US/Hawaii"),
+                    ("US/Indiana-Starke", "US/Indiana-Starke"),
+                    ("US/Michigan", "US/Michigan"),
+                    ("US/Mountain", "US/Mountain"),
+                    ("US/Pacific", "US/Pacific"),
+                    ("US/Samoa", "US/Samoa"),
+                    ("UTC", "UTC"),
+                    ("Universal", "Universal"),
+                    ("W-SU", "W-SU"),
+                    ("WET", "WET"),
+                    ("Zulu", "Zulu"),
+                ],
+                default="UTC",
+                max_length=255,
+            ),
+        ),
+    ]

bookwyrm/migrations/0171_merge_20221219_2020.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2022-12-19 20:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0167_sitesettings_import_size_limit"),
+        ("bookwyrm", "0170_merge_0168_auto_20221205_2331_0169_auto_20221206_0902"),
+    ]
+
+    operations = []

bookwyrm/migrations/0172_alter_user_preferred_language.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.16 on 2022-12-21 18:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0171_alter_user_preferred_timezone"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="user",
+            name="preferred_language",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("en-us", "English"),
+                    ("ca-es", "Català (Catalan)"),
+                    ("de-de", "Deutsch (German)"),
+                    ("es-es", "Español (Spanish)"),
+                    ("eu-es", "Euskara (Basque)"),
+                    ("gl-es", "Galego (Galician)"),
+                    ("it-it", "Italiano (Italian)"),
+                    ("fi-fi", "Suomi (Finnish)"),
+                    ("fr-fr", "Français (French)"),
+                    ("lt-lt", "Lietuvių (Lithuanian)"),
+                    ("no-no", "Norsk (Norwegian)"),
+                    ("pl-pl", "Polski (Polish)"),
+                    ("pt-br", "Português do Brasil (Brazilian Portuguese)"),
+                    ("pt-pt", "Português Europeu (European Portuguese)"),
+                    ("ro-ro", "Română (Romanian)"),
+                    ("sv-se", "Svenska (Swedish)"),
+                    ("zh-hans", "简体中文 (Simplified Chinese)"),
+                    ("zh-hant", "繁體中文 (Traditional Chinese)"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

bookwyrm/migrations/0173_author_website.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.16 on 2023-01-15 08:38
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0172_alter_user_preferred_language"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="website",
+            field=bookwyrm.models.fields.CharField(
+                blank=True, max_length=255, null=True
+            ),
+        ),
+    ]

bookwyrm/migrations/0173_default_user_auth_group_setting.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.16 on 2022-12-27 21:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def backfill_sitesettings(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+    group_model = apps.get_model("auth", "Group")
+    editor_group = group_model.objects.using(db_alias).filter(name="editor").first()
+
+    sitesettings_model = apps.get_model("bookwyrm", "SiteSettings")
+    sitesettings_model.objects.update(default_user_auth_group=editor_group)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0175_merge_0173_author_website_0174_merge_20230111_1523"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="default_user_auth_group",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.RESTRICT,
+                to="auth.group",
+            ),
+        ),
+        migrations.RunPython(backfill_sitesettings, migrations.RunPython.noop),
+    ]

bookwyrm/migrations/0173_merge_20230102_1444.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.16 on 2023-01-02 14:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0171_merge_20221219_2020"),
+        ("bookwyrm", "0172_alter_user_preferred_language"),
+    ]
+
+    operations = []