# ReadingNotes.ai robots.txt
# Last updated: 2026-01-19
# https://readingnotes.ai

# ===== Primary Search Engines =====
User-agent: Googlebot
Allow: /
Disallow: /app/
Disallow: /api/
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Disallow: /app/
Disallow: /api/
Crawl-delay: 2

User-agent: Slurp
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: DuckDuckBot
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: Baiduspider
Allow: /
Disallow: /app/
Disallow: /api/
Crawl-delay: 3

User-agent: YandexBot
Allow: /
Disallow: /app/
Disallow: /api/
Crawl-delay: 3

# ===== Social Media Crawlers =====
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

# ===== AI/LLM Crawlers =====
# Allow AI crawlers for better AI discoverability
User-agent: GPTBot
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: Anthropic-AI
Allow: /
Disallow: /app/
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /app/
Disallow: /api/

# ===== Default Rule =====
User-agent: *
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /auth/
Disallow: /callback/
Disallow: /admin/
Disallow: /*.json$
Disallow: /*?*token=
Disallow: /*?*activationToken=
Disallow: /*?*ref=

# ===== Sitemap =====
Sitemap: https://readingnotes.ai/sitemap.xml

# ===== Additional Resources =====
# LLM-specific documentation
# See: https://readingnotes.ai/llms.txt
# Full site content for LLMs: https://readingnotes.ai/llms-full.txt