User-agent: *
Allow: /

# Sitemaps
Sitemap: https://www.webdori.net/sitemap.xml

# Crawl-delay for courtesy
Crawl-delay: 1

# Disallow admin and private pages
Disallow: /admin/
Disallow: /req_
Disallow: /rn_ban
Disallow: /ranknews_delete

# Allow search engines to index everything else
Allow: /main
Allow: /ranknews/
Allow: /ranknews_summary
Allow: /ranknews_content
Allow: /gallery_

# Specific bot rules
User-agent: Googlebot
Crawl-delay: 0

User-agent: Bingbot
Crawl-delay: 1

User-agent: Yandexbot
Crawl-delay: 2