# ------------------------------------------------------------------
# Aggressive SEO / scraper bots: full disallow (they honor robots.txt,
# generate heavy CGI load, and bring no real visitors).
# ------------------------------------------------------------------
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: 008
Disallow: /

# ------------------------------------------------------------------
# Secondary crawlers: throttle.
# ------------------------------------------------------------------
User-agent: SeznamBot
Crawl-delay: 30

User-agent: Slurp
Crawl-delay: 10

User-agent: voyager
Crawl-delay: 20

User-agent: Yandex
Crawl-delay: 20

# ------------------------------------------------------------------
# Googlebot y Bingbot: mismas zonas prohibidas que el resto, pero SIN
# Crawl-delay. Google lo ignora de todas formas; Bing SI lo respeta, y
# con 10 s el techo eran 8.640 paginas al dia para las 17.681 URLs de
# los sitemaps: mas de dos dias para una sola pasada completa. El ritmo
# de Bing se ajusta desde Bing Webmaster Tools, no desde aqui.
# OJO: un grupo con nombre no hereda del grupo *, por eso se repiten
# los Disallow. Si se anade uno abajo, hay que anadirlo aqui tambien.
# ------------------------------------------------------------------
User-agent: Googlebot
Disallow: /indices/msgcenter/
Disallow: /indices/admin/
Disallow: /foros/admin/
Disallow: /admin/
Disallow: /city/panel.cgi
Disallow: /ciudadanos/
Disallow: /ajaxchat/
Disallow: /chat/
Disallow: /cgibin/registrar.cgi
Disallow: /cgibin/registro.cgi
Disallow: /cgibin/enviar_mail.cgi
Disallow: /postales/postcard.cgi
Disallow: /webalizer/
Disallow: /tebelogs/detalle.cgi

User-agent: Bingbot
Disallow: /indices/msgcenter/
Disallow: /indices/admin/
Disallow: /foros/admin/
Disallow: /admin/
Disallow: /city/panel.cgi
Disallow: /ciudadanos/
Disallow: /ajaxchat/
Disallow: /chat/
Disallow: /cgibin/registrar.cgi
Disallow: /cgibin/registro.cgi
Disallow: /cgibin/enviar_mail.cgi
Disallow: /postales/postcard.cgi
Disallow: /webalizer/
Disallow: /tebelogs/detalle.cgi

# ------------------------------------------------------------------
# Everyone else: crawl public content, keep out of admin / user /
# action / dynamic areas. Googlebot y Bingbot tienen grupo propio arriba;
# el Crawl-delay se queda aqui como freno para los bots sin nombre.
# ------------------------------------------------------------------
User-agent: *
Crawl-delay: 10
Disallow: /indices/msgcenter/
Disallow: /indices/admin/
Disallow: /foros/admin/
Disallow: /admin/
Disallow: /city/panel.cgi
Disallow: /ciudadanos/
Disallow: /ajaxchat/
Disallow: /chat/
Disallow: /cgibin/registrar.cgi
Disallow: /cgibin/registro.cgi
Disallow: /cgibin/enviar_mail.cgi
Disallow: /postales/postcard.cgi
Disallow: /webalizer/
Disallow: /tebelogs/detalle.cgi

# ------------------------------------------------------------------
# Sitemaps (uno por indice del motor; los regenera
# admin/scripts/sitemapsindices.cgi los martes a las 4:16).
# ------------------------------------------------------------------
Sitemap: https://dreamers.es/sitemaps/newssiteindex.xml
Sitemap: https://dreamers.es/sitemaps/peliculassiteindex.xml
Sitemap: https://dreamers.es/sitemaps/seriessiteindex.xml
Sitemap: https://dreamers.es/sitemaps/videojuegossiteindex.xml
Sitemap: https://dreamers.es/sitemaps/librossiteindex.xml
Sitemap: https://dreamers.es/sitemaps/comicssiteindex.xml
Sitemap: https://dreamers.es/sitemaps/discossiteindex.xml
Sitemap: https://dreamers.es/sitemaps/destacadossiteindex.xml
Sitemap: https://dreamers.es/sitemaps/articulossiteindex.xml
Sitemap: https://dreamers.es/sitemaps/autoressiteindex.xml
Sitemap: https://dreamers.es/sitemaps/personajessiteindex.xml
Sitemap: https://dreamers.es/sitemaps/relatossiteindex.xml