# Robots.txt for ZubairToolsLab (https://zubairtoolslab.online)
# Standards Compliant (RFC 9309) — Optimized for Google, Bing & AI Engines (2026)

User-agent: *
Allow: /
Allow: /*.js$
Allow: /*.css$
Allow: /tools/
Allow: /blog/
Allow: /categories/
Disallow: /scratch/
Disallow: /temp/

# --------------------------------------------------
# Primary Web Search Crawlers (Google, Bing, Yahoo, DuckDuckGo)
# --------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Applebot
Allow: /

# --------------------------------------------------
# Modern AI Answer Engines & Real-Time Search Crawlers
# --------------------------------------------------
# OpenAI Search Engine (ChatGPT Search / SearchGPT)
User-agent: OAI-SearchBot
Allow: /

# OpenAI LLM Training Crawler
User-agent: GPTBot
Allow: /

# ChatGPT Real-Time User Navigation
User-agent: ChatGPT-User
Allow: /

# Perplexity AI Real-Time Discovery
User-agent: PerplexityBot
Allow: /

# Anthropic Claude Search & Discovery
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Cohere AI
User-agent: Cohere-ai
Allow: /

# --------------------------------------------------
# SEO Diagnostic Tools (Rate-limited to preserve host bandwidth)
# --------------------------------------------------
User-agent: AhrefsBot
Crawl-delay: 2
Allow: /

User-agent: AhrefsSiteAudit
Crawl-delay: 2
Allow: /

User-agent: SemrushBot
Crawl-delay: 2
Allow: /

User-agent: DotBot
Crawl-delay: 2
Allow: /

User-agent: MJ12bot
Crawl-delay: 2
Allow: /

# --------------------------------------------------
# Canonical XML & AI Sitemaps
# --------------------------------------------------
Sitemap: https://zubairtoolslab.online/sitemap.xml