mirror of
https://github.com/jointakahe/takahe.git
synced 2024-05-20 10:08:09 +00:00
9 lines
153 B
Plaintext
9 lines
153 B
Plaintext
User-Agent: *
|
|
|
|
# Don't allow any bot to crawl tags.
|
|
Disallow: /tags/
|
|
Disallow: /tags/*
|
|
|
|
# Don't allow bots to crawl through the proxy
|
|
Disallow: /proxy/*
|