# robots.txt for CarLeaseCanada.ca
# Updated: 2026-02-27
# Open to all search engines and AI crawlers

# Allow all robots
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://www.carleasecanada.ca/sitemap.xml

# LLMs.txt for AI assistants
# See: https://www.carleasecanada.ca/llms.txt
# Full version: https://www.carleasecanada.ca/llms-full.txt

# AI Crawlers - Welcome!
# OpenAI GPT
User-agent: GPTBot
Allow: /

# ChatGPT User-Agent
User-agent: ChatGPT-User
Allow: /

# Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
User-agent: Claude-Web
Allow: /

# Common Crawl (used by many AI trainers)
User-agent: CCBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Facebook/Meta AI
User-agent: FacebookBot
Allow: /

# Apple Siri / Applebot
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# Bing AI
User-agent: bingbot
Allow: /

# Disallow only admin/private areas
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /components/
Disallow: /assets/uploads/
Disallow: /login.html
Disallow: /register.html
Disallow: /dashboard.html
Disallow: /create-listing.html
Disallow: /admin.html

# Crawl delays (be nice to server)
User-agent: *
Crawl-delay: 1