# https://www.robotstxt.org/robotstxt.html
User-agent: *
Allow: /

# The agent surface (RFC 0003) is programmatic, not a crawl target:
# the MCP transport and the REST API are rate-limited and answer JSON-RPC /
# JSON only, so crawling them wastes the caller's and our rate budgets.
Disallow: /mcp
Disallow: /api/

# Admin / dev / health
Disallow: /admin/
Disallow: /madmin/
Disallow: /sentry/
Disallow: /llm/
Disallow: /up
Disallow: /health

# Authenticated user areas
Disallow: /settings
Disallow: /profile
Disallow: /checkout
Disallow: /billing

# Auth endpoints (login/logout/password reset have no SEO value;
# token URLs in /passwords leak if indexed)
Disallow: /session
Disallow: /sign_in
Disallow: /sign_out
Disallow: /passwords/

# Email subscription pages and token URLs (no SEO value)
Disallow: /email_subscriptions/

# AI crawlers: explicit allowlist over the public site, with the agent
# surface excluded (same rule as *). Keeps the intent readable for crawler
# operators and future bots that ignore the * group.
User-agent: GPTBot
Allow: /
Disallow: /mcp
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /mcp
Disallow: /api/

User-agent: Claude-SearchBot
Allow: /
Disallow: /mcp
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /mcp
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /mcp
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /mcp
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /mcp
Disallow: /api/

# Sitemap location
Sitemap: https://foliofact.com/sitemap.xml