# ===============================
# ROBOTS.TXT
# Optimized for Google, Google News, OpenAI SearchBot, and SEO
# ===============================

# Allow OpenAI SearchBot
User-agent: OAI-SearchBot
Allow: /

# Googlebot (Web & News)
User-agent: Googlebot
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Disallow: /wp-content/plugins/
Disallow: /wp-includes/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /refer/
Disallow: /comments/feed/
Disallow: /wp-content/themes/*/includes/libs/better-framework/assets/fonts/

# Googlebot-News
User-agent: Googlebot-News
Allow: /
Disallow: /wp-admin/
Disallow: /wp-content/plugins/
Crawl-delay: 5

# Googlebot-Image (image indexing)
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Disallow: /wp-includes/

# General rules for all other bots
User-agent: *
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /refer/
Disallow: /comments/feed/
Crawl-delay: 20

# Aggressive / unwanted bots
User-agent: Yandex
Disallow: /

User-agent: amazon-kendra-web-crawler-*
Disallow: /

User-agent: Amazonbot
Disallow: /do-not-crawl/

User-agent: Baiduspider
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Seekport
Disallow: /

User-agent: SemrushBot-BM
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: macloud.host
Disallow: /

User-agent: macloud
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: bingbot
Crawl-delay: 10

User-agent: Slurp
Crawl-delay: 10