# Yobitel Communications — robots.txt
# All crawlers (incl. AI search engines) are welcome on public content.
# Auth/admin, API, and the CMS studio are disallowed for EVERY user-agent.
# NOTE: robots.txt user-agent blocks are not additive — a named bot ignores
# the `*` block entirely, so the same Disallow set is repeated per bot.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

# AI Search Crawlers — explicitly welcomed (same path restrictions apply)
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

# OpenAI ChatGPT Search crawler (distinct from GPTBot training crawler)
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

# Perplexity user-triggered fetch
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

# Apple Intelligence / Siri
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

# Common Crawl — feeds many downstream LLM training/answer sets
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /studio/
Disallow: /admin/
Disallow: /portal/

Sitemap: https://yobitel.com/sitemap.xml