# ============================================
# Robots.txt for YallaMotor
# Last updated: 2025-12-16
# ============================================

# ============================================
# 1. Block Known Bad Bots & Resource-Heavy Crawlers
# ============================================

User-agent: grapeshot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: WebReaper
Disallow: /

# Non-local or irrelevant regional bots
User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: MauiBot
Disallow: /

# ============================================


# ============================================
# OpenAI / LLM Crawlers
# ============================================

User-agent: OAI-SearchBot
Allow: /

User-agent: GPTBot
Allow: /

# ============================================
# Default Rules
# ============================================

User-agent: *
Disallow: /user/
Disallow: /admin/
Disallow: /filament/
Disallow: /conversations/
Disallow: /sales/
Disallow: /api/

# Old used-cars / sell form routes (noindex)
Disallow: /used-cars/sell-car
Disallow: /used-cars/sell-my-car/v2=1
Disallow: /used-cars/sell-my-car/
Disallow: /used-cars/car-information/
Disallow: /used-cars/upload-photos/
Disallow: /used-cars/contact-information/
Disallow: /used-cars/verify-mobile/
Disallow: /used-cars/payment-information/
Disallow: /used-cars/payfort-callback-url
Disallow: /used-cars/refresh-ad-payment/
Disallow: /used-cars/checkout-refresh-ad-payment/
Disallow: /ar/used-cars/sell-car
Disallow: /ar/used-cars/sell-my-car/v2=1
Disallow: /ar/used-cars/sell-my-car/
Disallow: /ar/used-cars/car-information/
Disallow: /ar/used-cars/upload-photos/
Disallow: /ar/used-cars/contact-information/
Disallow: /ar/used-cars/verify-mobile/
Disallow: /ar/used-cars/payment-information/
Disallow: /ar/used-cars/payfort-callback-url
Disallow: /ar/used-cars/refresh-ad-payment/
Disallow: /ar/used-cars/checkout-refresh-ad-payment/




# Block dynamic sort/filter/message URLs

Disallow: *?sort=*



# Block Dynamic Filter Messages & Cleaned State URLs

Disallow: *?message=*

Disallow: *&filters=*

# All other public-facing pages are allowed by default


# ============================================
# 3. Sitemap Location 
# ============================================

Sitemap: https://www.yallamotor.com/sitemap.xml