From 4b05dcde431ae7aa71dcd705358fbc2c7439db67 Mon Sep 17 00:00:00 2001
From: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Tue, 8 Aug 2023 13:16:34 +0200
Subject: [PATCH] [chore] Update robots.txt, give chatgpt the middle finger
 (#2085)

---
 internal/web/robots.go | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/internal/web/robots.go b/internal/web/robots.go
index b843203b..a79c3103 100644
--- a/internal/web/robots.go
+++ b/internal/web/robots.go
@@ -27,29 +27,43 @@ const (
 	robotsPath          = "/robots.txt"
 	robotsMetaAllowSome = "nofollow, noarchive, nositelinkssearchbox, max-image-preview:standard" // https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#robotsmeta
 	robotsTxt           = `# GoToSocial robots.txt -- to edit, see internal/web/robots.go
-# more info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro
+# More info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro
+
+# Before we commence, a giant fuck you to ChatGPT in particular.
+# https://platform.openai.com/docs/gptbot
+User-agent: GPTBot
+Disallow: /
+
+# Rules for everything else.
 User-agent: *
 Crawl-delay: 500
-# api stuff
+
+# API endpoints.
 Disallow: /api/
-# auth/login stuff
+
+# Auth/login endpoints.
 Disallow: /auth/
 Disallow: /oauth/
 Disallow: /check_your_email
 Disallow: /wait_for_approval
 Disallow: /account_disabled
-# well known stuff
+
+# Well-known endpoints.
 Disallow: /.well-known/
-# files
+
+# Fileserver/media.
 Disallow: /fileserver/
-# s2s AP stuff
+
+# Fedi S2S API endpoints.
 Disallow: /users/
 Disallow: /emoji/
-# panels
+
+# Settings panels.
 Disallow: /admin
 Disallow: /user
 Disallow: /settings/
-# domain blocklist
+
+# Domain blocklist.
 Disallow: /about/suspended`
 )