# SetDigger — https://www.setdigger.com
#
# Policy (carried forward from the retired apps/admin robots.txt decision log):
# welcome search engines and AI assistants; block abusive/low-value crawlers.
#
# - /login and /saved are deliberately NOT disallowed here: they carry
#   `noindex, nofollow` meta, and a robots.txt block would stop crawlers from
#   ever seeing that directive (blocked URLs can linger as URL-only listings).
# - /api/ is disallowed EXCEPT /api/og — the OG-image endpoint is referenced
#   as og:image/twitter:image on setlist, artist, and track pages, and
#   social/search crawlers honour robots.txt when fetching those images.
# - Aggressive SEO crawlers are blocked outright: negligible citation value,
#   and every uncached page render costs Worker CPU we cannot spare.
# - The block list is measurement-driven, not reputation-driven: a crawler
#   earns a block by generating high *cache-miss* volume (each miss is a Pages
#   Function invocation against the Free plan's fixed 10ms CPU budget) while
#   sending no traffic and producing no citations. Googlebot, bingbot,
#   Applebot, and the AI-assistant crawlers (ClaudeBot, GPTBot, OAI-SearchBot,
#   PerplexityBot, Amazonbot, meta-externalagent) stay welcome even though they
#   are also expensive — search and AI-assistant discoverability is the point
#   of the site's internal-crosslinking work, so their load is load we want.

User-agent: *
Allow: /api/og
Disallow: /api/
Crawl-delay: 1

# ─── Blocked: aggressive / low-value crawlers ────────────────────────────────
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: MJ12bot
Disallow: /

# Added after a cache-miss audit of live Cloudflare logs (2026-07-30). Counts
# below are cache MISSES per day — requests that reached the origin and burned
# Worker CPU.

# ~1,420 misses/day: the largest non-Google crawler on the site, against
# effectively no referred traffic. The render bot fetches page subresources on
# its own user-agent, so it needs its own block.
User-agent: YandexBot
Disallow: /

User-agent: YandexRenderResourcesBot
Disallow: /

# SEO-tooling crawlers in the same class as SemrushBot/AhrefsBot above: they
# index the catalogue to resell it, cite nothing back.
User-agent: serpstatbot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

Sitemap: https://www.setdigger.com/sitemap-index.xml