# manga-gastrica.mx - Robots.txt
# Dr. Cesar Hernandez Elenes - Bariatric Surgery

User-agent: *
Allow: /

# Allow all major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Disallow admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /private/
Disallow: /dashboard/

# Disallow search result pages to avoid duplicate content
Disallow: /search?
Disallow: /buscar?

# Disallow temporary or staging pages
Disallow: /temp/
Disallow: /staging/
Disallow: /test/

# Allow but crawl slowly for images and assets
User-agent: *
Crawl-delay: 1

# Specific rules for medical content crawlers
User-agent: MedicalBot
Allow: /
Crawl-delay: 2

# Block aggressive crawlers that might overload medical servers
User-agent: AhrefsBot
Crawl-delay: 5

User-agent: MJ12bot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

# Allow social media crawlers for better sharing
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Sitemap locations
Sitemap: https://manga-gastrica.mx/sitemap.xml

# Host specification (helps with canonicalization)
Host: https://manga-gastrica.mx