# Enhanced robots.txt for 2025 SEO & AI Crawlers
User-agent: *
Allow: /

# AI-Specific User Agents for 2025
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: AI2Bot
Allow: /

# Google AI crawlers
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Traditional Search Engine Bots
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

# Block malicious crawlers
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemap locations
Sitemap: https://barnamz.ir/sitemap.xml
Sitemap: https://barnamz.ir/sitemap-images.xml
Sitemap: https://barnamz.ir/sitemap-news.xml

# Crawl-delay for performance
Crawl-delay: 1

# Block access to specific files/directories
Disallow: /private/
Disallow: /admin/
Disallow: /temp/
Disallow: /backup/
Disallow: /logs/
Disallow: /*.json$
Disallow: /*.log$
Disallow: /404.html
Disallow: /500.html
Disallow: /*?*utm_*
Disallow: /*?*sid=*
Disallow: /*?*session*

# Allow access to important resources for AI understanding
Allow: /images/
Allow: /css/
Allow: /js/
Allow: /api/schema/
Allow: /manifest.json
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.avif$
Allow: /*.ico$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.pdf$

# Performance optimization directives
Request-rate: 1/10s

# Host preference for domain consolidation
Host: barnamz.ir

# Cache directive for better crawling efficiency
Cache-delay: 86400