# robots.txt, BroFindAI
# https://brofindai.com
#
# A crawler obeys only the most specific group that names it and ignores `*`
# entirely once it has one. So every named group below repeats the private
# paths: the old file gave Googlebot, Bingbot, GPTBot and the rest a bare
# "Allow: /", which quietly let them into /admin, /my-tools and /api/.

User-agent: *
Allow: /
Disallow: /admin
Disallow: /my-tools
Disallow: /saved
Disallow: /api/
Disallow: /.next/

# Google AdSense, the ad crawler needs to read the pages it serves ads on.
# Named explicitly so a future tightening of the `*` rules can't quietly cut
# off ad serving.
User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

# Search engines, AI assistants that cite sources, and backlink indexes.
#
#   Search:      Googlebot, GoogleOther, Bingbot, BingPreview, Brave, YouBot
#   AI answers:  GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot, Claude-Web,
#                anthropic-ai, PerplexityBot, Google-Extended, cohere-ai, CCBot
#   Backlinks:   AhrefsBot (Domain Rating), SemrushBot (Authority Score),
#                DotBot (Moz Domain Authority), MJ12bot (Majestic Trust Flow)
#
# The backlink indexes used to be blocked. That never changed how they score
# brofindai.com, but it hid every link this site gives out: a maker checking
# Ahrefs for the dofollow link their listing earned saw nothing, which
# undercuts the main reason makers list here and link back. The site is static
# on Cloudflare, so their crawl costs nothing.
User-agent: Googlebot
User-agent: GoogleOther
User-agent: Google-Extended
User-agent: Bingbot
User-agent: BingPreview
User-agent: Brave-Search
User-agent: YouBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: cohere-ai
User-agent: CCBot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
Allow: /
Disallow: /admin
Disallow: /my-tools
Disallow: /saved
Disallow: /api/
Disallow: /.next/

# Semrush's on-demand audit crawler, only useful to whoever runs the audit.
User-agent: SiteAuditBot
Disallow: /

Sitemap: https://brofindai.com/sitemap.xml

# Content usage preferences (https://contentsignals.org/)
# search=yes, allow indexing for search results
# ai-input=yes, allow use as AI system input / citations
# ai-train=no, do not use for AI model training without permission
Content-Signal: ai-train=no, search=yes, ai-input=yes