# Allow major search engines
User-agent: Googlebot
Disallow:

User-agent: SemrushBot
Disallow: 

User-agent: Bingbot
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: Yandex
Disallow:

# Disallow aggressive or unnecessary bots
User-agent: AwarioSmartBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Allow light and useful bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Catch-all for other bots
#User-agent: *
#Crawl-delay: 10
#Disallow: /wp-admin/
#Disallow: /wp-login.php
#Disallow: /xmlrpc.php
#Disallow: /?s=
#Disallow: /search
#Disallow: /?
#Disallow: /feed/
#Disallow: /mail.tau.edu.zm
#Sitemap: https://tau.edu.zm/sitemap_index.xml
# Throttle all other bots to reduce load
User-agent: *
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /search
Disallow: /feed/
# Allow friendly query strings (e.g. ?paged=2)
Disallow: /?xmlrpc.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /mail.tau.edu.zm
Disallow: /staging.tau.edu.zm

Sitemap: https://tau.edu.zm/sitemap_index.xml