# ######################################################################
# SEOTOLOGIE ROBOTS CONFIGURATION (robots.txt)
# ######################################################################
# Host     : https://seotologie.de
# Env      : Production
# Version  : 1.7.0 (Hardened & AI-Optimized)
# Stand    : September 2026
# System   : WordPress Standardized
# Author   : Peter Marth
# Contact  : info@seotologie.de
# Policy   : Strict Bot-Filter | AI-Rendering Allowed | SEO Scraper Throttled
# ######################################################################

# ==============================================================
# 1) KI-BOTS & SPEZIAL-CRAWLER (STRUKTURIERTER ZUGRIFF)
# ==============================================================
# Fokus: Maximale Sichtbarkeit in LLMs, Echtzeit-Suchen & Open-Source-KI.
# Erlaubt volles CSS/JS-Rendering, schuetzt aber die Server-Performance.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: OpenAI-Search
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Anthropic-ResearchBot
User-agent: PerplexityBot
User-agent: Applebot-Extended
User-agent: GoogleOther
User-agent: Google-Extended
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Meta-ExternalBot
User-agent: xai-bot
User-agent: HuggingFaceBot
User-agent: BraveBot
User-agent: CCBot
User-agent: YouBot
User-agent: CohereAI
User-agent: FacebookBot
User-agent: Diffbot
User-agent: Amazonbot
User-agent: SeznamBot
User-agent: MistralBot
User-agent: BingPreview
User-agent: CopilotBot
User-agent: DeepSeekBot
User-agent: DuckAssistBot
User-agent: Applebot

# --- Security Hardening fuer KI-Bots ---
Disallow: /private-security-log/
Disallow: /wp-content/private-security-log/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/languages/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /search/
Disallow: /attachment/
Disallow: /*/trackback/
Disallow: /*/print/
Disallow: /author/
Disallow: /*.sql$
Disallow: /*.log$

# --- WordPress Core-Assets (Rendering) ---
Allow: /wp-includes/*.js
Allow: /wp-includes/*.mjs
Allow: /wp-includes/*.css
Allow: /wp-includes/*.png
Allow: /wp-includes/*.jpg
Allow: /wp-includes/*.jpeg
Allow: /wp-includes/*.gif
Allow: /wp-includes/*.svg
Allow: /wp-includes/*.webp
Allow: /wp-includes/*.avif
Allow: /wp-includes/*.woff2
Allow: /wp-includes/*.woff
Allow: /wp-includes/*.ttf
Allow: /wp-includes/*.otf
Allow: /wp-includes/*.json

# --- Gezielte Parameter-Sperren ---

Disallow: /*?s=*
Disallow: /*?replytocom*


# ==============================================================
# 2) UNERWUENSCHTE CRAWLER & SPAM-BOTS (STRICT BLOCKED)
# ==============================================================
# Ziel: Komplette Aussperrung von Ressourcen-Fressern, Content-Dieben und Ad-Tech-Spionen.

User-agent: MJ12bot
User-agent: MJ12bot-v2
User-agent: MegaIndex.ru
User-agent: ZoominfoBot
User-agent: Mail.RU_Bot
User-agent: TrendictionBot
User-agent: ia_archiver
User-agent: Yandex
User-agent: Baiduspider
User-agent: DataForSeoBot
User-agent: AspiegelBot
User-agent: Exabot
User-agent: Sogou
User-agent: Nutch
User-agent: DotBot
User-agent: OmtrBot
User-agent: SeekportBot
User-agent: LinkpadBot
User-agent: AwarioBot
User-agent: AwarioSmartBot
User-agent: Scrapy
User-agent: CriteoBot
User-agent: GrapeshotCrawler
User-agent: PetalBot
User-agent: Bytespider
User-agent: Barkrowler
User-agent: TurnitinBot
User-agent: img2dataset
Disallow: /


# ==============================================================
# 3) SEO-TOOLS & ANALYSE-CRAWLER (THROTTLED & HARDENED)
# ==============================================================
# Ziel: Ausbremsen von SEO-Scrapern, Schutz vor endlosen Parameter-Loops im DACH- und Tech-Markt.

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: Sistrix
User-agent: XoviBot
User-agent: RyteBot
User-agent: SeobilityBot
User-agent: Screaming Frog SEO Spider
User-agent: SitebulbBot
User-agent: NetpeakSpiderBot
User-agent: Rogerbot
User-agent: Mozdotbot
User-agent: BLEXBot
User-agent: SerpstatBot
User-agent: SiteAuditBot
Crawl-delay: 5

# --- Core-Schutz vor SEO-Tools ---
Disallow: /private-security-log/
Disallow: /wp-content/private-security-log/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# --- Parameter-Sperre ---
Disallow: /*?*
Allow: /*?sitemap=*
Allow: /*?page=*

# --- Scraping-Drosselung ---
Disallow: /feed/


# ==============================================================
# 4) GLOBALE REGELN & FALLBACK (FUER GOOGLEBOT & REST)
# ==============================================================
User-agent: *

Disallow: /private-security-log/
Disallow: /wp-content/private-security-log/

# --- Gezielte Parameter-Sperren ---

Disallow: /*?s=*
Disallow: /*?replytocom*

# --- System-Isolierung ---
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /*?url=*
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/languages/
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /search/
Disallow: /attachment/
Disallow: /*/trackback/
Disallow: /*/print/
Disallow: /author/
Disallow: /*.sql$
Disallow: /*.log$

# --- Rendering-Ressourcen (SEO-BOOST) ---
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /wp-content/cache/

# --- WordPress Core-Assets ---
Allow: /wp-includes/*.js
Allow: /wp-includes/*.mjs
Allow: /wp-includes/*.css
Allow: /wp-includes/*.png
Allow: /wp-includes/*.jpg
Allow: /wp-includes/*.jpeg
Allow: /wp-includes/*.gif
Allow: /wp-includes/*.svg
Allow: /wp-includes/*.webp
Allow: /wp-includes/*.avif
Allow: /wp-includes/*.woff2
Allow: /wp-includes/*.woff
Allow: /wp-includes/*.ttf
Allow: /wp-includes/*.otf
Allow: /wp-includes/*.json

# API & Feed Sichtbarkeit
Allow: /wp-json/
Allow: /wp-json/wp/v2/
Allow: /feed/

# ==============================================================
# 5) SITEMAP-VERWEIS
# ==============================================================
Sitemap: https://seotologie.de/sitemap_index.xml