User-agent: *
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /docs
Disallow: /redoc
Disallow: /openapi.json
Disallow: /login
Disallow: /register
Disallow: /unsubscribe
Disallow: /reset-password
Disallow: /*.bak
Allow: /

# AI / GEO crawlers: allow citation and answer engines (do not block Googlebot).
# Curated facts for LLMs: https://magicmaster.pro/llms.txt
# Expanded reference: https://magicmaster.pro/llms-full.txt
# Playbook for agents that USE the service: https://magicmaster.pro/agents.md
# Machine-readable manifest: https://magicmaster.pro/agent.json

# One group for all AI crawlers (RFC 9309: consecutive User-agent lines share rules).
# A named group REPLACES the * group for that bot, so the Disallow list is repeated here —
# with only "Allow: /" these bots were free to crawl /api/ and /admin (audit 25.09).
# Private pages (/wallet, /profile, /dashboard) are closed by meta noindex, not here:
# a page disallowed in robots is never read, so its noindex is never seen.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: Google-Extended
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: CCBot
User-agent: Bytespider
User-agent: cohere-ai
User-agent: DuckAssistBot
User-agent: YouBot
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /docs
Disallow: /redoc
Disallow: /openapi.json
Disallow: /login
Disallow: /register
Disallow: /unsubscribe
Disallow: /reset-password
Disallow: /*.bak
Allow: /

# Fetches a person asked for (not crawling): an agent using the service may read the
# API description (/openapi.json, /docs) — only the admin area is closed.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
Disallow: /admin
Disallow: /admin/
Disallow: /*.bak
Allow: /

User-agent: Yandex
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /docs
Disallow: /redoc
Disallow: /openapi.json
Disallow: /login
Disallow: /register
Disallow: /unsubscribe
Disallow: /reset-password
Disallow: /*.bak
Allow: /
# Яндекс: трекинг-параметры — дубли; lang НЕ включаем (hreflang / SSR ?lang=en).
Clean-param: ref&utm_source&utm_medium&utm_campaign&utm_content&utm_term&fbclid&gclid&yclid&_openstat&from&ymclid&plan&rt&aiclean&intent&style&src

Sitemap: https://magicmaster.pro/sitemap.xml