# robots.txt for abacktools.com
#
# Policy (one statement, applied consistently — see docs/audits/ P0-4):
#   - Search indexing:      allowed
#   - AI input / grounding: allowed  (we publish llms.txt, llms-full.txt,
#                                    ai.txt, /tools.json and /docs/*.md for
#                                    agents to read)
#   - Model training:       NOT allowed
#
# Rationale: we want to be found and cited. We do not want our content used to
# train models. `ai-input=yes` and `ai-train=no` are not in conflict — they say
# "read this at query time, don't memorise it", which is exactly what our
# machine-readable files are for. The previous file said ai-input=no while
# simultaneously inviting crawlers in, so neither signal could be trusted.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/

# Opt out of model-training crawlers.
# These do not affect Google Search or Bing ranking, and they do not affect the
# search/retrieval and user-triggered agents listed further down, which are
# what actually drive AI answers.
#
# ClaudeBot is Anthropic's training crawler, so `ai-train=no` puts it here next
# to GPTBot and CCBot. Anthropic's search crawler (Claude-SearchBot) and its
# user-triggered fetcher (Claude-User) are separate agents and stay allowed.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Timpibot
Disallow: /

# Deliberately NOT listed as named groups, so the `*` rules above (including
# Disallow: /api/) apply to them:
#
#   Search engines:
#     Googlebot, Bingbot, Applebot, YandexBot, DuckDuckBot, Baiduspider, Naver,
#     Seznam, ia_archiver.
#
#   AI search / retrieval crawlers (index pages so answers can cite them; they
#   do not train models):
#     OAI-SearchBot (ChatGPT search), Claude-SearchBot (Claude search),
#     PerplexityBot (Perplexity search), YouBot.
#
#   User-triggered agents (fetch a page because a person asked their assistant
#   to; they act for that person, not as crawlers):
#     ChatGPT-User, Claude-User, Perplexity-User.
#
# A named group containing only "Allow: /" silently overrides every `*` rule,
# which is how the old file ended up with /_next/ blocked for Bing, Yandex,
# DuckDuckGo, Baidu, Naver and Seznam but not for Google. Bing's index feeds
# Copilot and ChatGPT search, so a broken CSS/JS render was being evaluated for
# layout and mobile-friendliness.

# Sitemaps
Sitemap: https://abacktools.com/sitemap.xml

# Agentic Resource Discovery manifest (ARD spec section 6.1). Lists the MCP
# server, OpenAPI description, agent skill and llms.txt.
Agentmap: https://abacktools.com/.well-known/ai-catalog.json

# AI-native discovery files. These are intentionally NOT disallowed above:
# they exist to be read by agents, and `ai-input=yes` says so. All of them are
# generated from the tool registry at build time.
#   https://abacktools.com/llms.txt
#   https://abacktools.com/llms-full.txt
#   https://abacktools.com/ai.txt
#   https://abacktools.com/tools.json            (machine-readable catalog)
#   https://abacktools.com/agents/policy.json    (prefill/autorun policy)
#   https://abacktools.com/docs/index.md
#   https://abacktools.com/docs/agents.md        (how an agent finds and operates a tool)
#   https://abacktools.com/.well-known/agent-skills/index.json
#   https://abacktools.com/.well-known/api-catalog         (RFC 9727)
#   https://abacktools.com/.well-known/mcp/server-card.json (MCP server at /api/mcp)