# SEO / marketing crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Barkrowler
Disallow: /

# Crawling tools / scrapers
User-agent: Screaming Frog
Disallow: /

User-agent: ScreamingFrogSEOSpider
Disallow: /

User-agent: Sitebulb
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WinHTTrack
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: ZyBorg
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: ltx71
Disallow: /

User-agent: GeedoShopProductFinder
Disallow: /

# Regional / alternative search engines
User-agent: PetalBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: SeznamBot
Disallow: /

# Misc / aggressive / unknown
User-agent: Buck/2.3.2
Disallow: /

User-agent: TurnitinBot
Disallow: /

# Default rules for all other crawlers (Google, Bing, etc.)
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Additional rules preserved from existing robots.txt
Sitemap: https://youngatart.co.uk/sitemap_index.xml