User-agent: *
Disallow: /wp-content/uploads/sites/11/job_applications/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Rate limit all crawlers (honored by Bing, Yandex, etc.; ignored by GPTBot/Google)
Crawl-delay: 10

# AI crawlers (GPTBot, OAI-SearchBot, ChatGPT-User, CCBot, Amazonbot,
# ClaudeBot, PerplexityBot, Google-Extended, etc.) are intentionally NOT listed
# here, so they inherit the User-agent: * rules above: full content access,
# admin paths blocked.
# NOTE: a named "User-agent: <bot>" group fully REPLACES the wildcard group for
# that bot (groups are never merged). If you add one, repeat the Disallow rules
# above or you will expose /wp-admin/ to that bot.
# Crawl-delay is advisory and ignored by GPTBot, Googlebot, and Amazonbot, so it
# is NOT origin protection. Rate-limit abusive crawl in nginx (limit_req).

# Block aggressive / abusive bots entirely
# Bytespider stays blocked: it routinely ignores Crawl-delay and hammers origin.
User-agent: Sogou
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Sitemap
Sitemap: https://dirtcar.com/sitemap_index.xml