User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Allow: /wp-admin/admin-ajax.php


# Slow down some aggressive bots (polite ones will respect this)
User-agent: AhrefsBot
Crawl-delay: 5

User-agent: DotBot
Crawl-delay: 5

User-agent: MJ12bot
Crawl-delay: 5

# Block shady scrapers that do respect robots.txt (some do)
User-agent: Python-Requests
Disallow: /

User-agent: python
Disallow: /

User-agent: curl
Disallow: /