# robots.txt - Project Insider Asia
# https://www.projectinsiderasia.com/robots.txt
#
# RULE ORDER CRITICAL:
#   1. Group-specific User-agent blocks FIRST (most specific)
#   2. Catch-all User-agent: * LAST (least specific)
# Crawlers stop at the FIRST matching User-agent block they find.
# If * is first, every crawler reads only the * block and ignores
# all the Googlebot / Bingbot / GPTBot blocks below it.

# ─── GOOGLE CRAWLERS ───
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Google-Extended
Allow: /

# ─── BING / MICROSOFT CRAWLERS ───
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: MSNBot
Allow: /

# ─── YAHOO CRAWLERS ───
User-agent: Slurp
Allow: /
Crawl-delay: 1

# ─── DUCKDUCKGO ───
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# ─── AI / LLM CRAWLERS ───
# Allowing these ensures the site appears in AI search results
# (ChatGPT, Perplexity, Claude, etc.)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# ─── SOCIAL MEDIA CRAWLERS ───
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: WhatsApp
Allow: /

# ─── SEO / AUDIT TOOLS ───
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

User-agent: Screaming Frog SEO Spider
Allow: /
Crawl-delay: 2

# ─── CATCH-ALL (must be LAST) ───
User-agent: *
Allow: /
# Internal platform-transparency tooling: noindexed and excluded from the
# XML sitemap; not part of the public content surface.
Disallow: /dashboard/

# ─── SITEMAP ───
Sitemap: https://www.projectinsiderasia.com/sitemap.xml

# Security contact: see /.well-known/security.txt (RFC 9116)