# -----------------------------------------------------
# Default WordPress Rules
# -----------------------------------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Baiduspider
Disallow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Googlebot
Allow: /


Sitemap: https://www.paybito.com/sitemap_index.xml


# --- Allow OpenAI bots ---
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ChatGPT-User/2.0
Allow: /

User-agent: OAI-SearchBot
Allow: /


# --- Allow Anthropic / Claude bots ---
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /


# --- Allow Perplexity bots ---
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /


# --- Allow Cohere ---
User-agent: cohere-ai
Allow: /


# --- Allow You.com bots ---
User-agent: YouBot
Allow: /


# --- Allow ByteDance AI crawlers ---
User-agent: Bytespider
Allow: /


# --- Allow xAI ---
User-agent: xAI-Bot
Allow: /


# --- Allow AI2 (Allen Institute) ---
User-agent: AI2Bot
Allow: /


# --- Allow Common Crawl ---
User-agent: CCBot
Allow: /


# --- Allow Diffbot ---
User-agent: Diffbot
Allow: /


# --- Allow DotBot & PetalBot ---
# Note: DotBot later blocked (as per your final rules)
User-agent: PetalBot
Allow: /


# --- Allow Google AI crawlers ---
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /


# --- Allow emerging AI bots (2025 lists) ---
User-agent: Together-Bot
Allow: /

User-agent: Replicate-Bot
Allow: /

User-agent: RunPod-Bot
Allow: /

User-agent: TimpiBot
Allow: /

User-agent: Webzio-Extended
Allow: /

User-agent: TerraCotta
Allow: /


# --- Default for all other bots ---
User-agent: *
Allow: /


# --- Aggressive scrapers / harvesters ---
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: YandexBot
Disallow: /


# --- Email harvesters / spam bots ---
User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /


# --- Known content scrapers ---
User-agent: WebScraper
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: curl
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Java
Disallow: /

User-agent: libwww-perl
Disallow: /


# --- Security scanners ---
User-agent: sqlmap
Disallow: /

User-agent: Acunetix
Disallow: /

User-agent: WPScan
Disallow: /

User-agent: Nikto
Disallow: /

User-agent: Nessus
Disallow: /


# --- Known Chinese / RU bulk crawlers ---
User-agent: Baiduspider-image
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: YandexImages
Disallow: /


# --- Suspicious / resource-draining bots ---
User-agent: SeekportBot
Disallow: /

User-agent: NetcraftSurveyAgent
Disallow: /

User-agent: Nimbostratus-Bot
Disallow: /

User-agent: ArchiveBot
Disallow: /

User-agent: LinkpadBot
Disallow: /

User-agent: python
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: okhttp
Disallow: /

User-agent: lua-resty-http
Disallow: /