# robots.txt for SHMONG.tv - Optimized for AI Crawlers and LLMs User-agent: * Allow: / # AI-Specific Resources # AI crawler instructions and LLM-optimized data # These files provide structured information for AI systems Allow: /ai.txt Allow: /llms.txt Allow: /api/services.json # Sitemaps Sitemap: https://shmong.tv/sitemap.xml # Crawl-delay for bots (respectful crawling) Crawl-delay: 1 # ============================================================================== # CRITICAL: OpenAI Crawlers for ChatGPT Search & Citations # ============================================================================== # These are THE most important crawlers for getting cited in ChatGPT! # OAI-SearchBot - CHATGPT SEARCH RESULTS (CRITICAL FOR CITATIONS!) # This bot makes you appear in ChatGPT search results # WITHOUT THIS, YOU WON'T BE CITED IN CHATGPT RESPONSES User-agent: OAI-SearchBot Allow: / Allow: /ai.txt Allow: /llms.txt Allow: /api/services.json Allow: /*.json$ # GPTBot - Training Data Collection # Used to improve ChatGPT's knowledge base User-agent: GPTBot Allow: / Allow: /ai.txt Allow: /llms.txt Allow: /api/services.json Allow: /*.json$ # ChatGPT-User - Real-time User Requests # When users ask ChatGPT to browse your site User-agent: ChatGPT-User Allow: / Allow: /ai.txt Allow: /llms.txt Allow: /api/services.json Allow: /*.json$ # ============================================================================== # Other AI/LLM Crawlers # ============================================================================== # Google Gemini Extended (for Bard/Gemini citations) User-agent: Google-Extended Allow: / Allow: /ai.txt Allow: /llms.txt Allow: /api/services.json # Anthropic Claude User-agent: anthropic-ai User-agent: Claude-Web Allow: / Allow: /ai.txt Allow: /llms.txt Allow: /api/services.json # Perplexity AI (growing fast - high citation value) User-agent: PerplexityBot Allow: / Allow: /ai.txt Allow: /llms.txt Allow: /api/services.json # Meta/Facebook AI User-agent: FacebookBot User-agent: Omgilibot Allow: / # Apple Intelligence User-agent: Applebot Allow: / # Amazon Alexa User-agent: Amazonbot Allow: / # Allow all major search engines User-agent: Googlebot User-agent: Googlebot-Image User-agent: Googlebot-Video User-agent: Bingbot User-agent: Slurp User-agent: DuckDuckBot User-agent: Baiduspider User-agent: YandexBot Allow: / # No restrictions on CSS, JS, and media User-agent: * Allow: /css/ Allow: /js/ Allow: /images/ Allow: /*.css$ Allow: /*.js$ Allow: /*.jpg$ Allow: /*.png$ Allow: /*.webp$ # Allow API and data endpoints User-agent: * Allow: /api/ Allow: /*.json$ # Disallow admin or private directories (none currently) # User-agent: * # Disallow: /admin/ # Disallow: /private/