Stop storing activitypub json in db entirely

fedireads/activity.py

@@ -1,7 +1,14 @@
 ''' Handle user activity '''
+from base64 import b64encode
+from Crypto.PublicKey import RSA
+from Crypto.Signature import pkcs1_15
+from Crypto.Hash import SHA256
+from uuid import uuid4
+
 from fedireads import models
 from fedireads.openlibrary import get_or_create_book
 from fedireads.sanitize_html import InputHtmlParser
+from fedireads.settings import DOMAIN
 
 
 def create_review(user, possible_book, name, content, rating):
@@ -17,7 +24,7 @@ def create_review(user, possible_book, name, content, rating):
     # no ratings outside of 0-5
     rating = rating if 0 <= rating <= 5 else 0
 
-    review = models.Review.objects.create(
+    return models.Review.objects.create(
         user=user,
         book=book,
         name=name,
@@ -25,6 +32,108 @@ def create_review(user, possible_book, name, content, rating):
         content=content,
     )
 
-    return review
+
+def create_status(user, content, reply_parent=None):
+    ''' a status update '''
+    # TODO: handle @'ing users
+
+    # sanitize input html
+    parser = InputHtmlParser()
+    parser.feed(content)
+    content = parser.get_output()
+
+    return models.Status.objects.create(
+        user=user,
+        content=content,
+        reply_parent=reply_parent,
+    )
+
+
+def get_status_json(status):
+    ''' create activitypub json for a status '''
+    user = status.user
+    uri = 'https://%s/user/%s/status/%d' % (DOMAIN, user.localname, status.id)
+    reply_parent_id = status.reply_parent.id if status.reply_parent else None
+    status_json = {
+        'id': uri,
+        'url': uri,
+        'inReplyTo': reply_parent_id,
+        'published': status.created_date.isoformat(),
+        'attributedTo': user.actor,
+        # TODO: assuming all posts are public -- should check privacy db field
+        'to': ['https://www.w3.org/ns/activitystreams#Public'],
+        'cc': ['https://%s/user/%s/followers' % (DOMAIN, user.localname)],
+        'sensitive': status.sensitive,
+        'content': status.content,
+        'type': status.activity_type,
+        'fedireadsType': status.status_type,
+        'attachment': [], # TODO: the book cover
+        'replies': {
+            'id': '%s/replies' % uri,
+            'type': 'Collection',
+            'first': {
+                'type': 'CollectionPage',
+                'next': '%s/replies?only_other_accounts=true&page=true' % uri,
+                'partOf': '%s/replies' % uri,
+                'items': [], # TODO: populate with replies
+            }
+        }
+    }
+
+    if status.status_type == 'Review':
+        status_json['name'] = status.name,
+        status_json['rating'] = status.rating
+        status_json['fedireadsType'] = status.status_type
+
+    return status_json
+
+
+def get_create_json(user, status_json):
+    ''' create activitypub json for a Create activity '''
+    signer = pkcs1_15.new(RSA.import_key(user.private_key))
+    content = status_json['content']
+    signed_message = signer.sign(SHA256.new(content.encode('utf8')))
+    return {
+        '@context': 'https://www.w3.org/ns/activitystreams',
+
+        'id': '%s/activity' % status_json['id'],
+        'type': 'Create',
+        'actor': user.actor,
+        'published': status_json['published'],
+
+        'to': ['%s/followers' % user.actor],
+        'cc': ['https://www.w3.org/ns/activitystreams#Public'],
+
+        'object': status_json,
+        'signature': {
+            'type': 'RsaSignature2017',
+            'creator': 'https://%s/user/%s#main-key' % (DOMAIN, user.localname),
+            'created': status_json['published'],
+            'signatureValue': b64encode(signed_message).decode('utf8'),
+        }
+    }
+
+
+
+def get_add_remove_json(user, book, shelf, action='Add'):
+    ''' format an Add or Remove json blob '''
+    uuid = uuid4()
+    return {
+        '@context': 'https://www.w3.org/ns/activitystreams',
+        'id': str(uuid),
+        'type': action,
+        'actor': user.actor,
+        'object': {
+            'type': 'Document',
+            'name': book.data['title'],
+            'url': book.openlibrary_key
+        },
+        'target': {
+            'type': 'Collection',
+            'name': shelf.name,
+            'id': 'https://%s/user/%s/shelf/%s' % \
+                (DOMAIN, user.localname, shelf.identifier)
+        }
+    }
 
 

fedireads/incoming.py

@@ -11,7 +11,7 @@ import requests
 
 from fedireads import models
 from fedireads import outgoing
-from fedireads.activity import create_review
+from fedireads.activity import create_review, create_status
 from fedireads.remote_user import get_or_create_remote_user
 
 
@@ -33,10 +33,7 @@ def shared_inbox(request):
         return HttpResponse(status=401)
 
     response = HttpResponseNotFound()
-    if activity['type'] == 'Add':
-        response = handle_incoming_shelve(activity)
-
-    elif activity['type'] == 'Follow':
+    if activity['type'] == 'Follow':
         response = handle_incoming_follow(activity)
 
     elif activity['type'] == 'Create':
@@ -45,7 +42,7 @@ def shared_inbox(request):
     elif activity['type'] == 'Accept':
         response = handle_incoming_follow_accept(activity)
 
-    # TODO: Undo, Remove, etc
+    # TODO: Add, Undo, Remove, etc
 
     return response
 
@@ -217,43 +214,12 @@ def get_follow_page(user_list, id_slug, page):
     return data
 
 
-def handle_incoming_shelve(activity):
-    ''' receiving an Add activity (to shelve a book) '''
-    # TODO what happens here? If it's a remote over, then I think
-    # I should save both the activity and the ShelfBook entry. But
-    # I'll do that later.
-    uuid = activity['id']
-    models.ShelveActivity.objects.get(uuid=uuid)
-    '''
-    book_id = activity['object']['url']
-    book = openlibrary.get_or_create_book(book_id)
-    user_ap_id = activity['actor'].replace('https//:', '')
-    user = models.User.objects.get(actor=user_ap_id)
-    if not user or not user.local:
-        return HttpResponseBadRequest()
-
-    shelf = models.Shelf.objects.get(activitypub_id=activity['target']['id'])
-    models.ShelfBook(
-        shelf=shelf,
-        book=book,
-        added_by=user,
-    ).save()
-    '''
-    return HttpResponse()
-
-
 def handle_incoming_follow(activity):
     ''' someone wants to follow a local user '''
     # figure out who they want to follow
     to_follow = models.User.objects.get(actor=activity['object'])
     # figure out who they are
     user = get_or_create_remote_user(activity['actor'])
-    models.FollowActivity(
-        uuid=activity['id'],
-        user=user,
-        followed=to_follow,
-        content=activity,
-    )
     # TODO: allow users to manually approve requests
     outgoing.handle_outgoing_accept(user, to_follow, activity)
     return HttpResponse()
@@ -278,36 +244,27 @@ def handle_incoming_create(activity):
         return HttpResponseBadRequest()
 
     response = HttpResponse()
+    content = activity['object'].get('content')
     if activity['object'].get('fedireadsType') == 'Review' and \
             'inReplyTo' in activity['object']:
         book = activity['object']['inReplyTo']
         book = book.split('/')[-1]
         name = activity['object'].get('name')
-        content = activity['object'].get('content')
         rating = activity['object'].get('rating')
         if user.local:
             review_id = activity['object']['id'].split('/')[-1]
-            review = models.Review.objects.get(id=review_id)
+            models.Review.objects.get(id=review_id)
         else:
             try:
-                review = create_review(user, book, name, content, rating)
+                create_review(user, book, name, content, rating)
             except ValueError:
                 return HttpResponseBadRequest()
-        models.ReviewActivity.objects.create(
-            uuid=activity['id'],
-            user=user,
-            content=activity['object'],
-            activity_type=activity['object']['type'],
-            book=review.book,
-        )
-
     else:
-        models.Activity.objects.create(
-            uuid=activity['id'],
-            user=user,
-            content=activity,
-            activity_type=activity['object']['type']
-        )
+        try:
+            create_status(user, content)
+        except ValueError:
+            return HttpResponseBadRequest()
+
     return response
 
 
@@ -321,12 +278,5 @@ def handle_incoming_accept(activity):
     # save this relationship in the db
     followed.followers.add(user)
 
-    # save the activity record
-    models.FollowActivity(
-        uuid=activity['id'],
-        user=user,
-        followed=followed,
-        content=activity,
-    ).save()
     return HttpResponse()
 

fedireads/models/__init__.py

@@ -1,6 +1,5 @@
 ''' bring all the models into the app namespace '''
 from .book import Shelf, ShelfBook, Book, Author
 from .user import User, FederatedServer
-from .activity import Activity, ShelveActivity, FollowActivity, \
-     ReviewActivity, Status, Review
+from .activity import Status, Review
 

fedireads/models/activity.py

@@ -3,66 +3,15 @@ from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from model_utils.managers import InheritanceManager
 
-from fedireads.utils.fields import JSONField
-
-# TODO: I don't know that these Activity models should exist, at least in this way
-# but I'm not sure what the right approach is for now.
-
-class Activity(models.Model):
-    ''' basic fields for storing activities '''
-    uuid = models.CharField(max_length=255, unique=True)
-    user = models.ForeignKey('User', on_delete=models.PROTECT)
-    content = JSONField(max_length=5000)
-    # the activitypub activity type (Create, Add, Follow, ...)
-    activity_type = models.CharField(max_length=255)
-    # custom types internal to fedireads (Review, Shelve, ...)
-    fedireads_type = models.CharField(max_length=255, blank=True, null=True)
-    local = models.BooleanField(default=True)
-    created_date = models.DateTimeField(auto_now_add=True)
-    updated_date = models.DateTimeField(auto_now=True)
-    objects = InheritanceManager()
-
-
-class ShelveActivity(Activity):
-    ''' someone put a book on a shelf '''
-    book = models.ForeignKey('Book', on_delete=models.PROTECT)
-    shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT)
-
-    def save(self, *args, **kwargs):
-        if not self.activity_type:
-            self.activity_type = 'Add'
-        self.fedireads_type = 'Shelve'
-        super().save(*args, **kwargs)
-
-
-class FollowActivity(Activity):
-    ''' record follow requests sent out '''
-    followed = models.ForeignKey(
-        'User',
-        related_name='followed',
-        on_delete=models.PROTECT
-    )
-
-    def save(self, *args, **kwargs):
-        self.activity_type = 'Follow'
-        super().save(*args, **kwargs)
-
-
-class ReviewActivity(Activity):
-    book = models.ForeignKey('Book', on_delete=models.PROTECT)
-
-    def save(self, *args, **kwargs):
-        self.activity_type = 'Note'
-        self.fedireads_type = 'Review'
-        super().save(*args, **kwargs)
-
 
 class Status(models.Model):
-    ''' reply to a review, etc '''
+    ''' any post, like a reply to a review, etc '''
     user = models.ForeignKey('User', on_delete=models.PROTECT)
     status_type = models.CharField(max_length=255, default='Note')
-    activity = JSONField(max_length=5000, null=True)
+    activity_type = models.CharField(max_length=255, default='Note')
     local = models.BooleanField(default=True)
+    privacy = models.CharField(max_length=255, default='public')
+    sensitive = models.BooleanField(default=False)
     reply_parent = models.ForeignKey(
         'self',
         null=True,
@@ -70,14 +19,13 @@ class Status(models.Model):
     )
     content = models.TextField(blank=True, null=True)
     created_date = models.DateTimeField(auto_now_add=True)
-    updated_date = models.DateTimeField(auto_now=True)
     objects = InheritanceManager()
 
 
 class Review(Status):
     ''' a book review '''
-    book = models.ForeignKey('Book', on_delete=models.PROTECT)
     name = models.CharField(max_length=255)
+    book = models.ForeignKey('Book', on_delete=models.PROTECT)
     rating = models.IntegerField(
         default=0,
         validators=[MinValueValidator(0), MaxValueValidator(5)]
@@ -87,4 +35,3 @@ class Review(Status):
         self.status_type = 'Review'
         super().save(*args, **kwargs)
 
-

fedireads/outgoing.py

@@ -1,9 +1,4 @@
 ''' handles all the activity coming out of the server '''
-from base64 import b64encode
-from Crypto.PublicKey import RSA
-from Crypto.Signature import pkcs1_15
-from Crypto.Hash import SHA256
-from datetime import datetime
 from django.http import HttpResponseNotFound, JsonResponse
 from django.views.decorators.csrf import csrf_exempt
 import requests
@@ -11,7 +6,8 @@ from urllib.parse import urlencode
 from uuid import uuid4
 
 from fedireads import models
-from fedireads.activity import create_review
+from fedireads.activity import create_review, get_status_json, get_create_json
+from fedireads.activity import get_add_remove_json
 from fedireads.remote_user import get_or_create_remote_user
 from fedireads.broadcast import get_recipients, broadcast
 from fedireads.settings import DOMAIN
@@ -43,19 +39,17 @@ def outbox(request, username):
             filters['id__lte'] = max_id
         collection_id = query_path + urlencode(params)
 
-        messages = models.Activity.objects.filter(
-            user=user,
-            activity_type__in=['Article', 'Note'],
-            **filters
-            ).all()[:limit]
-
         outbox_page = {
             '@context': 'https://www.w3.org/ns/activitystreams',
             'id': collection_id,
             'type': 'OrderedCollectionPage',
             'partOf': user.outbox,
-            'orderedItems': [m.content for m in messages],
+            'orderedItems': [],
         }
+        statuses = models.Status.objects.filter(user=user, **filters).all()
+        for status in statuses[:limit]:
+            outbox_page['orderedItems'].append(get_status_json(status))
+
         if max_id:
             outbox_page['next'] = query_path + \
                 urlencode({'min_id': max_id, 'page': 'true'})
@@ -65,7 +59,7 @@ def outbox(request, username):
         return JsonResponse(outbox_page)
 
     # collection overview
-    size = models.Review.objects.filter(user=user).count()
+    size = models.Status.objects.filter(user=user).count()
     return JsonResponse({
         '@context': 'https://www.w3.org/ns/activitystreams',
         'id': '%s/outbox' % user.actor,
@@ -134,90 +128,25 @@ def handle_outgoing_accept(user, to_follow, activity):
 def handle_shelve(user, book, shelf):
     ''' a local user is getting a book put on their shelf '''
     # update the database
+    # TODO: this should probably happen in incoming instead
     models.ShelfBook(book=book, shelf=shelf, added_by=user).save()
 
-    # send out the activitypub action
-    summary = '%s marked %s as %s' % (
-        user.username,
-        book.data['title'],
-        shelf.name
-    )
-
-    uuid = uuid4()
-    activity = {
-        '@context': 'https://www.w3.org/ns/activitystreams',
-        'id': str(uuid),
-        'summary': summary,
-        'type': 'Add',
-        'actor': user.actor,
-        'object': {
-            'type': 'Document',
-            'name': book.data['title'],
-            'url': book.openlibrary_key
-        },
-        'target': {
-            'type': 'Collection',
-            'name': shelf.name,
-            'id': 'https://%s/user/%s/shelf/%s' % \
-                (DOMAIN, user.localname, shelf.identifier)
-        }
-    }
+    activity = get_add_remove_json(user, book, shelf, 'Add')
     recipients = get_recipients(user, 'public')
 
-    models.ShelveActivity(
-        uuid=uuid,
-        user=user,
-        content=activity,
-        shelf=shelf,
-        book=book,
-    ).save()
-
     broadcast(user, activity, recipients)
 
 
 def handle_unshelve(user, book, shelf):
     ''' a local user is getting a book put on their shelf '''
     # update the database
+    # TODO: this should probably happen in incoming instead
     row = models.ShelfBook.objects.get(book=book, shelf=shelf)
     row.delete()
 
-    # send out the activitypub action
-    summary = '%s removed %s from %s' % (
-        user.username,
-        book.data['title'],
-        shelf.name
-    )
-
-    uuid = uuid4()
-    activity = {
-        '@context': 'https://www.w3.org/ns/activitystreams',
-        'id': str(uuid),
-        'summary': summary,
-        'type': 'Remove',
-        'actor': user.actor,
-        'object': {
-            'type': 'Document',
-            'name': book.data['title'],
-            'url': book.openlibrary_key
-        },
-        'target': {
-            'type': 'Collection',
-            'name': shelf.name,
-            'id': 'https://%s/user/%s/shelf/%s' % \
-                (DOMAIN, user.localname, shelf.identifier)
-        }
-    }
+    activity = get_add_remove_json(user, book, shelf, 'Remove')
     recipients = get_recipients(user, 'public')
 
-    models.ShelveActivity(
-        uuid=uuid,
-        user=user,
-        content=activity,
-        shelf=shelf,
-        book=book,
-        activity_type='Remove',
-    ).save()
-
     broadcast(user, activity, recipients)
 
 
@@ -226,63 +155,10 @@ def handle_review(user, book, name, content, rating):
     # validated and saves the review in the database so it has an id
     review = create_review(user, book, name, content, rating)
 
-    review_path = 'https://%s/user/%s/status/%d' % \
-        (DOMAIN, user.localname, review.id)
-    book_path = 'https://%s/book/%s' % (DOMAIN, review.book.openlibrary_key)
+    #book_path = 'https://%s/book/%s' % (DOMAIN, review.book.openlibrary_key)
 
-    now = datetime.utcnow().isoformat() #TODO: should this be http_date?
-    review_activity = {
-        'id': review_path,
-        'url': review_path,
-        'inReplyTo': book_path,
-        'published': now,
-        'attributedTo': user.actor,
-        # TODO: again, assuming all posts are public
-        'to': ['https://www.w3.org/ns/activitystreams#Public'],
-        'cc': ['https://%s/user/%s/followers' % (DOMAIN, user.localname)],
-        'sensitive': False, # TODO: allow content warning/sensitivity
-        'content': content,
-        'type': 'Note',
-        'fedireadsType': 'Review',
-        'name': name,
-        'rating': rating, # fedireads-only custom field
-        'attachment': [], # TODO: the book cover
-        'replies': {
-            'id': '%s/replies' % review_path,
-            'type': 'Collection',
-            'first': {
-                'type': 'CollectionPage',
-                'next': '%s/replies?only_other_accounts=true&page=true' % \
-                    review_path,
-                'partOf': '%s/replies' % review_path,
-                'items': [], # TODO: populate with replies
-            }
-        }
-    }
-    review.activity = review_activity
-    review.save()
-
-    signer = pkcs1_15.new(RSA.import_key(user.private_key))
-    signed_message = signer.sign(SHA256.new(content.encode('utf8')))
-    create_activity = {
-        '@context': 'https://www.w3.org/ns/activitystreams',
-
-        'id': '%s/activity' % review_path,
-        'type': 'Create',
-        'actor': user.actor,
-        'published': now,
-
-        'to': ['%s/followers' % user.actor],
-        'cc': ['https://www.w3.org/ns/activitystreams#Public'],
-
-        'object': review_activity,
-        'signature': {
-            'type': 'RsaSignature2017',
-            'creator': 'https://%s/user/%s#main-key' % (DOMAIN, user.localname),
-            'created': now,
-            'signatureValue': b64encode(signed_message).decode('utf8'),
-        }
-    }
+    review_activity = get_status_json(review)
+    create_activity = get_create_json(user, review_activity)
 
     recipients = get_recipients(user, 'public')
     broadcast(user, create_activity, recipients)

fedireads/templates/feed.html

@@ -55,40 +55,21 @@
         <h2>
             {% include 'snippets/avatar.html' with user=activity.user %}
             {% include 'snippets/username.html' with user=activity.user %}
-        {% if activity.fedireads_type == 'Shelve' %}
-            {# display a reading/shelving activity #}
-            {% if activity.shelf.identifier == 'to-read' %}
-                wants to read
-            {% elif activity.shelf.identifier == 'read' %}
-                finished reading
-            {% elif activity.shelf.identifier == 'reading' %}
-                started reading
-            {% else %}
-                shelved in "{{ activity.shelf.name }}"
-            {% endif %}
-        </h2>
-            <div class="book-preview">
-                {% include 'snippets/book.html' with book=activity.book size=large description=True %}
-            </div>
-            <div class="interaction"><button>⭐️ Like</button></div>
-        {% elif activity.fedireads_type == 'Review' %}
+        {% if activity.status_type == 'Review' %}
             {# display a review #}
             reviewed {{ activity.book.data.title }}
         </h2>
             <div class="book-preview review">
                 {% include 'snippets/book.html' with book=activity.book size=large %}
 
-                <h3>{{ activity.content.name }}</h3>
-                <p>{{ activity.content.rating | stars }}</p>
-                <p>{{ activity.content.content | safe }}</p>
+                <h3>{{ activity.name }}</h3>
+                <p>{{ activity.rating | stars }}</p>
+                <p>{{ activity.content | safe }}</p>
             </div>
             <div class="interaction"><button>⭐️ Like</button></div>
-        {% elif activity.activity_type == 'Follow' %}
-            started following someone
-        </h2>
-        {% elif activity.activity_type == 'Note' %}
+        {% elif activity.status_type == 'Note' %}
             posted</h2>
-            {{ activity.content.object.content | safe }}
+            {{ activity.content | safe }}
         {% else %}
         {# generic handling for a misc activity, which perhaps should not be displayed at all #}
             did {{ activity.activity_type }}

fedireads/views.py

@@ -39,7 +39,7 @@ def home(request):
     # TODO: this is fundamentally not how the feed should work I think? it
     # should do something smart with inboxes. (in this implementation it would
     # show DMs meant for other local users)
-    activities = models.Activity.objects.filter(
+    activities = models.Status.objects.filter(
         user__in=following,
     ).select_subclasses().order_by(
         '-created_date'