# ═══════════════════════════════════════════════════════════════
# centraldev.az robots.txt — search-engine crawl rules
# Last updated: 2026-09-08
# ═══════════════════════════════════════════════════════════════

# ── Default: all bots allowed ──────────────────────────────────
User-agent: *
Allow: /

# Block form endpoints (POST-only)
Disallow: /eleqe/gonder
Disallow: /dil/
Disallow: /admin/
Disallow: /admin

# Block Laravel internals (defence in depth)
Disallow: /storage/
Disallow: /vendor/
Disallow: /.env
Disallow: /composer.json
Disallow: /composer.lock

# ── Prefer Google's canonical crawl (max 3 sec between requests)
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /img/
Allow: /public/img/

# ── AI crawlers: allow (be discoverable) ──────────────────────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

# ── Aggressive/junk crawlers: throttle ────────────────────────
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

# ── Sitemaps ──────────────────────────────────────────────────
Sitemap: https://centraldev.az/sitemap.xml