# Агрессивные SEO-краулеры и скраперы
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Majestic-12
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Dotbot
Disallow: /

User-agent: Seekport
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: AlphaBot
Disallow: /

User-agent: LinkpadBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: linkdex
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: ExtLinksBot
Disallow: /

User-agent: WebDataStats
Disallow: /

User-agent: WebPulse
Disallow: /

User-agent: Lipperhey
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: WeSEE:Search
Disallow: /

User-agent: SearchmetricsBot
Disallow: /

User-agent: spbot
Disallow: /

# Бот OpenAI (GPTBot) - блокировка сбора данных для обучения
User-agent: GPTBot
Disallow: /

# Общие боты, которые могут создавать нагрузку
User-agent: CCBot
Crawl-delay: 10

User-agent: Applebot
Crawl-delay: 5

User-agent: Bingbot
Crawl-delay: 5

# Официальные поисковые системы (разрешаем с задержкой)
User-agent: Googlebot
Crawl-delay: 2
Allow: /

User-agent: YandexBot
Crawl-delay: 2
Allow: /

User-agent: Baiduspider
Crawl-delay: 10
Allow: /

User-agent: Sogou web spider
Crawl-delay: 10
Allow: /

User-agent: *
Disallow: /*?
Disallow: /admin/
Disallow: /cards-info/
Disallow: /sedi-meztaxi.html
Disallow: /vakansii
Disallow: /politi*
Host: https://www.mezhgorod.taxi/
Sitemap: https://www.mezhgorod.taxi/sitemapindex.xml