# ==========================================================
# DAMIAN SCHMIDT | GLOBAL SEO & AI ARCHITECTURE
# Optimized for: AIo, GEO, and Agentic Search
# Last Updated: 2026-02-17
# ==========================================================

User-agent: *
# Default rule: Allow all high-quality search engines
Allow: /
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /private/
Disallow: /error-logs/
# Prevent crawling of specific technical or temporary files
Disallow: /*.php$
Disallow: /*.inc$

# ----------------------------------------------------------
# AI & GENERATIVE ENGINE OPTIMIZATION (GEO)
# Explicitly inviting the primary LLM training crawlers
# ----------------------------------------------------------

User-agent: GPTBot
Allow: / # OpenAI (ChatGPT)

User-agent: ChatGPT-User
Allow: / # OpenAI (Web Browsing)

User-agent: Google-Extended
Allow: / # Google Gemini / SGE

User-agent: PerplexityBot
Allow: / # Perplexity AI

User-agent: ClaudeBot
Allow: / # Anthropic Claude

User-agent: OAI-SearchBot
Allow: / # OpenAI Search

User-agent: CCBot
Allow: / # Common Crawl (Used for training most LLMs)

# ----------------------------------------------------------
# RESOURCE MANAGEMENT
# Managing aggressive SEO tools and scrapers
# ----------------------------------------------------------

User-agent: AhrefsBot
Crawl-delay: 2

User-agent: DotBot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 2

# ----------------------------------------------------------
# SECURITY & METADATA
# ----------------------------------------------------------

# Security Policy discoverability
Allow: /.well-known/security.txt

# Humans.txt for developer transparency
Allow: /humans.txt

# Global Sitemap
Sitemap: https://www.damianschmidt.com/sitemap.xml

# Final Directive: Block known malicious/useless scrapers
User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /