User-agent: *
Allow: /

# Sitemap
Sitemap: https://audits.blockhacks.io/sitemap.xml

# High priority pages
Allow: /
Allow: /api/rss


# Additional rules for better SEO
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Chinese search engines
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: Sogou
Allow: /
Crawl-delay: 2

User-agent: 360Spider
Allow: /
Crawl-delay: 2

# OpenAI crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: CCBot
Allow: /
Crawl-delay: 1

# Disallow admin or sensitive areas if any
# Disallow: /admin
# Disallow: /api/private

# General crawl delay for respectful crawling
Crawl-delay: 1