# Allow AI search and agent use
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
Allow: /
 
# Disallow AI training data collection
User-agent: GPTBot
User-agent: CCBot
User-agent: Google-Extended
Disallow: /
 
# Allow traditional and global search engines
User-agent: Googlebot
User-agent: Bingbot
User-agent: Yandex
User-agent: Baiduspider
User-agent: DuckDuckBot
User-agent: EcosiaBot
User-agent: Sogou web spider
User-agent: Yeti
User-agent: SeznamBot
Allow: /
 
# Disallow access to admin areas for all bots
User-agent: *
Disallow: /admin/
Disallow: /internal/
 
Sitemap: https://sciflare.com/sitemap.xml