# robots.txt for remoote.app
# Updated: 2026-05-19
# Location: S3 web bucket (remoote-web-production), served via CloudFront
# Deploy: aws s3 cp robots.txt s3://remoote-web-production/robots.txt --cache-control "max-age=86400"

# =============================================================================
# SEARCH / INDEX / CITATION CRAWLERS
# Clean pages are allowed; faceted/filter URL combinations are blocked for all.
# =============================================================================
User-agent: *
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: facebookexternalhit
Allow: /

# Block query parameter variations (faceted navigation)
Disallow: /*?*job_id=
Disallow: /*?*skill_ids=
Disallow: /*?*categories=
Disallow: /*?*company_sizes=

# Block authentication and admin
Disallow: /auth
Disallow: /auth/
Disallow: /admin
Disallow: /admin/
Disallow: /api
Disallow: /api/

# Block legacy/unfinished static pages
Disallow: /jobs/title
Disallow: /jobs/skill
Disallow: /jobs/country

# =============================================================================
# AI TRAINING CRAWLERS (BLOCK - these train models, not search)
# Uncomment to block if you don't want content used for training
# =============================================================================
User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Bytespider
Disallow: /

# =============================================================================
# SITEMAP & LLMs.txt
# =============================================================================
Sitemap: https://remoote.app/sitemap/sitemap.xml

# Preferred AI agent access and MCP/API docs
# See: https://remoote.app/agents

# LLM-readable site overview (llms.txt standard)
# See: https://remoote.app/llms.txt