# DebtShark Robots.txt
# Canonical URL: https://boredpolymath.github.io/debtshark/
# Machine-readable AI & LLM Guidance: https://boredpolymath.github.io/debtshark/llms.txt
# Full LLM Context: https://boredpolymath.github.io/debtshark/llms-full.txt

# ==============================================================================
# AI Assistants & Large Language Model Crawlers
# Permitted to crawl and index all public documentation and LLM context files
# ==============================================================================
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Diffbot
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
Allow: /
Allow: /debtshark/
Allow: /llms.txt
Allow: /debtshark/llms.txt
Allow: /llms-full.txt
Allow: /debtshark/llms-full.txt
Allow: /sitemap.xml
Allow: /debtshark/sitemap.xml
Allow: /site.webmanifest
Allow: /debtshark/site.webmanifest
Allow: /manifest.json
Allow: /debtshark/manifest.json
Allow: /.well-known/security.txt
Allow: /debtshark/.well-known/security.txt
Allow: /humans.txt
Allow: /debtshark/humans.txt
Allow: /favicon.ico
Allow: /debtshark/favicon.ico

# ==============================================================================
# Search Engine Web Crawlers
# ==============================================================================
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
Allow: /
Allow: /debtshark/

# ==============================================================================
# Social Media Link Preview Crawlers
# ==============================================================================
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: TelegramBot
User-agent: Discordbot
Allow: /
Allow: /debtshark/

# ==============================================================================
# Default Rule for All Other Crawlers
# ==============================================================================
User-agent: *
Allow: /
Allow: /debtshark/
Crawl-delay: 1

# ==============================================================================
# Sitemaps & Context Feeds
# ==============================================================================
Sitemap: https://boredpolymath.github.io/debtshark/sitemap.xml