User-agent: *
Allow: /

# Sitemap
Sitemap: https://havahavai.com/sitemap.xml

# Disallow crawling of API routes and internal files
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /.well-known/
Disallow: /private/

# Allow crawling of all public content
Allow: /blog/
Allow: /images/
Allow: /favicon.ico
Allow: /manifest.json
Allow: /ai.txt
Allow: /humans.txt

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Specific directives for AI crawlers
User-agent: GPTBot
Allow: /
Allow: /blog/
Allow: /ai.txt
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Allow: /blog/
Allow: /ai.txt

User-agent: CCBot
Allow: /
Allow: /blog/
Allow: /ai.txt

User-agent: anthropic-ai
Allow: /
Allow: /blog/
Allow: /ai.txt

User-agent: Claude-Web
Allow: /
Allow: /blog/
Allow: /ai.txt