User-agent: *
Allow: /

# ===========================================
# PRIVATE / USER-SPECIFIC PAGES
# ===========================================
# Account dashboard and settings (user-specific, no SEO value)
Disallow: /account
Disallow: /account/

# User profile pages (privacy - users may not want to be indexed)
Disallow: /profile/

# ===========================================
# ADMIN PAGES
# ===========================================
Disallow: /admin
Disallow: /admin/

# ===========================================
# AUTHENTICATION PAGES
# ===========================================
# Auth callbacks and password reset (sensitive, transient pages)
Disallow: /auth/
Disallow: /auth/callback
Disallow: /auth/reset-password

# ===========================================
# CONTENT CREATION / EDITING PAGES
# ===========================================
# Blog writing page (requires login, no SEO value)
Disallow: /blogs/write

# Classified creation/edit pages
Disallow: /classifieds/create
Disallow: /*/classifieds/create
Disallow: /*/classifieds/edit/

# Recommendation creation/edit pages
Disallow: /recommendations/create
Disallow: /*/recommendations/create
Disallow: /*/recommendations/edit/

# Question ask/edit pages
Disallow: /questions/ask
Disallow: /*/questions/ask
Disallow: /*/questions/edit/

# Business listing submission form (requires login)
Disallow: /list-business

# NOTE: /add-your-business is ALLOWED - it's a marketing landing page

# ===========================================
# API ENDPOINTS
# ===========================================
# Block all API routes (except sitemap)
Disallow: /api/

# Explicitly allow sitemap API
Allow: /api/sitemap

# ===========================================
# UTILITY / SYSTEM PAGES
# ===========================================
# Search results pages (dynamic, duplicate content, wastes crawl budget)
Disallow: /search
Disallow: /*?q=
Disallow: /*?search=
Disallow: /*?query=

# Session/tracking parameters only
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?source=

# ===========================================
# SITEMAP & AI GUIDANCE
# ===========================================
Sitemap: https://expatslist.org/sitemap.xml

# LLMs.txt - AI-specific site information
# See: https://expatslist.org/llms.txt
# See also: https://expatslist.org/llms-full.txt

# ===========================================
# MAJOR SEARCH ENGINES - WELCOME
# ===========================================
# Google
User-agent: Googlebot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap
Disallow: /profile/
Disallow: /blogs/write
Disallow: /classifieds/create
Disallow: /*/classifieds/create
Disallow: /*/classifieds/edit/
Disallow: /recommendations/create
Disallow: /*/recommendations/create
Disallow: /*/recommendations/edit/
Disallow: /questions/ask
Disallow: /*/questions/ask
Disallow: /*/questions/edit/
Disallow: /list-business
Disallow: /search
# NOTE: /add-your-business is ALLOWED for Googlebot

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

# Bing
User-agent: Bingbot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap
Disallow: /profile/
Disallow: /blogs/write
Disallow: /classifieds/create
Disallow: /*/classifieds/create
Disallow: /*/classifieds/edit/
Disallow: /recommendations/create
Disallow: /*/recommendations/create
Disallow: /*/recommendations/edit/
Disallow: /questions/ask
Disallow: /*/questions/ask
Disallow: /*/questions/edit/
Disallow: /list-business

User-agent: msnbot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/

# Yandex
User-agent: Yandex
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap
Disallow: /profile/
Disallow: /blogs/write
Disallow: /classifieds/create
Disallow: /*/classifieds/create
Disallow: /*/classifieds/edit/
Disallow: /recommendations/create
Disallow: /*/recommendations/create
Disallow: /*/recommendations/edit/
Disallow: /questions/ask
Disallow: /*/questions/ask
Disallow: /*/questions/edit/
Disallow: /list-business

User-agent: YandexBot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

User-agent: YandexImages
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# Baidu (China)
User-agent: Baiduspider
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# ===========================================
# AI BOTS - WELCOME TO INDEX PUBLIC CONTENT
# ===========================================
# OpenAI bots (ChatGPT, GPT search)
User-agent: GPTBot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

User-agent: ChatGPT-User
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# Anthropic bots (Claude)
User-agent: anthropic-ai
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

User-agent: Claude-Web
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# ClaudeBot (Anthropic web crawler for Claude AI search)
User-agent: ClaudeBot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# Google AI (Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# Apple AI (Siri, Apple Intelligence)
User-agent: Applebot-Extended
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# Microsoft Copilot
User-agent: OAI-SearchBot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# Meta AI
User-agent: meta-externalagent
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap

# Common Crawl (used by many AI training datasets)
User-agent: CCBot
Allow: /
Disallow: /account
Disallow: /admin
Disallow: /auth/
Disallow: /api/
Allow: /api/sitemap