# robots.txt for graveshow.com
# The Graveshow's Eye — https://graveshow.com
# Updated: May 2026

# ── GENERAL CRAWLERS ────────────────────────────────────────
User-agent: *
Allow: /

# Allow llms.txt explicitly for all agents
Allow: /llms.txt

# Disallow WordPress admin and system paths
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-cron.php
Disallow: /wp-trackback.php
Disallow: /?s=
Disallow: /search/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /author/
Disallow: /page/
Disallow: /?p=
Disallow: /?page_id=
Disallow: /tag/

# ── AI & LLM CRAWLERS ───────────────────────────────────────

# OpenAI (ChatGPT, GPT-4 etc.)
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/

# OpenAI image / content crawler
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/

# Anthropic extended crawler
User-agent: Claude-Web
Allow: /
Allow: /llms.txt

# Google Gemini / Bard
User-agent: Google-Extended
Allow: /
Allow: /llms.txt

# Google general
User-agent: Googlebot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Bing / Microsoft Copilot
User-agent: Bingbot
Allow: /
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /wp-includes/

# Microsoft Copilot crawler
User-agent: msnbot
Allow: /
Allow: /llms.txt

# Meta AI
User-agent: FacebookBot
Allow: /
Allow: /llms.txt

# Apple
User-agent: Applebot
Allow: /
Allow: /llms.txt

# Common Crawl (used to train many LLMs)
User-agent: CCBot
Allow: /
Allow: /llms.txt

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt

# You.com
User-agent: YouBot
Allow: /
Allow: /llms.txt

# Cohere
User-agent: cohere-ai
Allow: /
Allow: /llms.txt

# ── UNWANTED BOTS ───────────────────────────────────────────

# Scrapers and content thieves
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ── SITEMAPS & DISCOVERY ────────────────────────────────────
Sitemap: https://graveshow.com/wp-sitemap.xml
LLMs: https://graveshow.com/llms.txt