# workx.am - Armenia's #1 Job Board
# Strategy: Allow search engines + AI search bots, block training-only bots

# ─── Default rules for all bots ───
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /dashboard/*
Disallow: /onboarding
Disallow: /onboarding/*
Disallow: /settings
Disallow: /settings/*
Disallow: /admin
Disallow: /admin/*
Disallow: /payments
Disallow: /payments/*
Disallow: /api/
# Cloudflare internal paths (email-protection links etc.) - crawlers 404 on
# these; blocking stops the noise in Search Console's Not-found report.
Disallow: /cdn-cgi/
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /email/
Disallow: /two-factor-challenge
# /login and /register: NOT blocked here - the React pages already
# emit <meta name="robots" content="noindex">. Blocking via robots.txt
# would prevent Google from seeing the noindex tag, so the pages would
# get indexed anyway from inbound links (the "Indexed, though blocked
# by robots.txt" warning in GSC). Letting crawlers see the noindex tag
# is the only way to actually drop these URLs.

# ─── AI Search/Retrieval Bots (ALLOW) ───
# These power AI search results - allowing them means workx.am
# appears when users ask ChatGPT, Perplexity, etc. about jobs in Armenia

# OpenAI - powers ChatGPT search
User-agent: ChatGPT-User
Allow: /

# Anthropic - powers Claude search
User-agent: ClaudeBot
Allow: /

# Perplexity AI search
User-agent: PerplexityBot
Allow: /

# Amazon - powers Alexa/Amazon search
User-agent: Amazonbot
Allow: /

# You.com AI search
User-agent: YouBot
Allow: /

# Cohere AI search
User-agent: cohere-ai
Allow: /

# Apple AI (Siri, Apple Intelligence)
User-agent: Applebot
Allow: /

# Microsoft Bing (also powers Copilot)
User-agent: Bingbot
Allow: /

# ─── AI Training Bots (ALLOW for visibility) ───
# These crawl content for model training. Allowing them increases
# the chance workx.am knowledge appears in future AI models.
# Block these if you prefer to opt out of AI training.

# OpenAI training crawler
User-agent: GPTBot
Allow: /

# Google AI training (Gemini)
User-agent: Google-Extended
Allow: /

# Anthropic training crawler
User-agent: anthropic-ai
Allow: /

# Meta AI training
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ─── Aggressive/Unwanted Bots (BLOCK or THROTTLE) ───

# ByteDance - known for extremely aggressive crawling
User-agent: Bytespider
Disallow: /

# Diffbot - scraper
User-agent: Diffbot
Disallow: /

# img2dataset - image scraper
User-agent: img2dataset
Disallow: /

# Common Crawl - massive dataset crawler (allow for AI training inclusion)
User-agent: CCBot
Allow: /

# SEO tools - throttle to prevent server overload
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

# ─── MCP Server (AI Agent API) ───
# AI agents can connect via MCP protocol at:
# https://workx.am/mcp
# Discovery: https://workx.am/.well-known/mcp.json

# ─── Sitemaps ───
Sitemap: https://workx.am/sitemap.xml
Sitemap: https://workx.am/feeds/jobs.rss