# Parked and non-public routes. /merch also redirects to the home page and
# carries a noindex tag; this keeps well-behaved crawlers off it entirely.
User-agent: *
Disallow: /merch
Disallow: /merch/
Allow: /

# Internet Archive / Wayback Machine. These are the crawler names the Archive
# has used: ia_archiver is the legacy one, archive.org_bot the current save
# and crawl bot, and Heritrix the crawler behind both. Note this is a request,
# not a control - since 2017 the Archive has said it largely disregards
# robots.txt for archiving. To have pages actually removed or kept out, write
# to info@archive.org; this file only stops the crawlers that still honour it.
User-agent: ia_archiver
Disallow: /

User-agent: ia_archiver-web.archive.org
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: Wayback
Disallow: /

User-agent: heritrix
Disallow: /