# GetMattered - robots.txt
# https://getmattered.com
# AEO discovery files: /llms.txt, /llms-full.txt, /llms-context.md, /pricing.md,
# /ai.txt, /api/ai.json
#
# Every page has exactly ONE url in every language. There is no locale path
# segment and no locale subdomain: /pricing serves German to a German reader
# and English to an English one, and both emit the same canonical. The five
# files above negotiate their language the same way.
#
# Generated from the same policy tables the edge enforces. If a crawler is
# named below, the worker treats it identically - the two cannot drift.

User-agent: *
# Content Signals (contentsignals.org): what automated systems may DO with
# content they are already allowed to fetch. Preference, not access control.
# Content-Signal: search=yes, ai-input=yes, ai-train=yes (policy published in /ai.txt and /llms.txt)

# Allow public pages
Allow: /
Allow: /about
Allow: /ai-disclosure
Allow: /alternatives/
Allow: /api/ai.json
Allow: /best/
Allow: /blog/
Allow: /blog/rss.xml
Allow: /contact
Allow: /creator-program
Allow: /faq
Allow: /legal/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /mcps
Allow: /pricing
Allow: /pricing.md
Allow: /sitemap.xml
Allow: /sitemap-index.xml
Allow: /sitemap-blog.xml
Allow: /sitemap-static.xml
Allow: /sitemap-vs.xml
Allow: /vs/

# Disallow protected and private content
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

# No Crawl-delay anywhere in this file, by design. Google has never supported
# the field, and Bing and Yandex read its absence as exactly the unrestricted
# crawl that an explicit zero would have asked for. Crawl as fast as you like.

# ============================================================================
# SEARCH + ANSWER ENGINES - crawl as fast as you like.
# These find us, index us, and cite us. Never rate-limited, never blocked,
# never challenged, on any surface.
# ============================================================================

User-agent: Googlebot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Google-Extended
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: GoogleOther
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Google-InspectionTool
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Bingbot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: BingPreview
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: GPTBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: OAI-SearchBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: ChatGPT-User
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: ChatGPT Agent
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: PerplexityBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Perplexity-User
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Comet
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: ClaudeBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Claude-Web
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Claude-User
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Claude-SearchBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: anthropic-ai
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: DuckAssistBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: DuckDuckBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: MistralAI-User
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: cohere-ai
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: YouBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Slurp
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Applebot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: YandexBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Baiduspider
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: PetalBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: SeznamBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Yeti
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: meta-externalfetcher
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

# ============================================================================
# AI TRAINING CRAWLERS - also welcome, same access, no crawl delay.
# ============================================================================

User-agent: meta-externalagent
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: FacebookBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Amazonbot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Bytespider
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: TikTokSpider
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: CCBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: Diffbot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: AwarioBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

User-agent: HeadlessChrome
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

# ============================================================================
# SEO / BACKLINK CRAWLERS - allowed, same access as everyone else.
# ============================================================================

User-agent: AhrefsBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: AhrefsSiteAudit
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: SemrushBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: MJ12bot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: DataForSeoBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: SERankingBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: DotBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: rogerbot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: BLEXBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: Barkrowler
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: serpstatbot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: linkdexbot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: SEOkicks
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: SiteCheckerBotCrawler
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: ZoominfoBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: omgili
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: webzio-extended
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: ImagesiftBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: Timpibot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

User-agent: stepstoneCrawlBot
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=
Disallow: /jobs/
Disallow: /companies/

# ============================================================================
# APPLE AI TRAINING - EXPLICITLY WELCOMED (own group; does not inherit Applebot)
# ============================================================================

User-agent: Applebot-Extended
Allow: /api/ai.json
Disallow: /api/
Disallow: /api/v1/csp-report
Disallow: /auth/
Disallow: /dashboard
Disallow: /admin/
Disallow: /billing
Disallow: /_protected/
Disallow: /_serverFn/
Disallow: /_build/
Disallow: /_stream/
Disallow: /ingest/
Disallow: /settings/
Disallow: /health
Disallow: /*?utm_*
Disallow: /*?gclid=*
Disallow: /*?fbclid=*
Disallow: /*_fresh=
Disallow: /*_sb=
Disallow: /*_sbr=
Disallow: /*_build_version=

# Sitemaps (declared at end so crawlers parse all group rules first)
#
# The index is advertised under a FRESH URL, /sitemap-index.xml, followed by
# the three SERVING children. /sitemap-index.xml and /sitemap.xml serve the
# SAME <sitemapindex> (both routes proxy kind 'index'); only the name differs.
#
# Why the fresh name: GSC reports "Discovered pages" PER SUBMITTED SITEMAP
# URL, and that number is a lifetime union of everything ever found through
# that URL string. /sitemap.xml has carried ~11,440 since it advertised the
# retired ~11K /jobs/{id} family, and the same URL does NOT reset:
#   - 2026-08-10: full DELETE (204) + re-PUT (204) + same-day re-download came
#     straight back to 11,440 while the children updated within the minute.
#   - 2026-09-05: PUT-only resubmit of all four re-downloaded every child within
#     3 seconds (static 26 -> 27); the index re-downloaded and read 11,440 again.
#   - 2026-09-05: sitemaps.get still returns the DELETED sitemap-jobs.xml record
#     and 404s for sitemap-index.xml, so Google keys the record by URL string
#     and this name carries no history.
# Google documents an index row as the sum of its children, computed
# asynchronously, so the new row is expected to read what the three children
# sum to that day. If it reads 11,440 instead, the counter is keyed by child
# history; the extra line is harmless and there is no third reset.
#
# ⛔ NO PER-SITEMAP URL COUNT IS WRITTEN HERE ON PURPOSE. This comment used to
# carry "(25 + 18 + 153 = 196)"; by 2026-09-05 the live files served 27 + 4 +
# 151 = 182 and the /vs figure was out by 4.5x. A hardcoded count in a file
# that nothing recomputes is a stale instruction, not a fact. Read the live
# children, or "bun run check:seo", which walks the advertised index and
# prints every child every run.
#
# /sitemap.xml still SERVES (conventional path, other crawlers fetch it by
# habit) and is still deliberately NOT advertised here, so Google stops
# re-discovering the stale counter through it.
Sitemap: https://getmattered.com/sitemap-index.xml
Sitemap: https://getmattered.com/sitemap-static.xml
Sitemap: https://getmattered.com/sitemap-vs.xml
Sitemap: https://getmattered.com/sitemap-blog.xml