# ─────────────────────────────────────────────────────────────────────────────
# robots.txt — GTM North
# Stance: maximum AI distribution (same as the Wix site, which already allowed
# GPTBot, ClaudeBot and Google-Extended). Search engines, AI-search citation
# crawlers and training crawlers are all allowed; Content-Signal below states it.
#
# Each crawler obeys ONLY its most specific matching group, so the Disallow set
# is repeated in every group on purpose. Keep the blocks identical.
#
# Cloudflare's managed "Block AI bots" robots.txt injection must be OFF on the
# zone, or it prepends Disallow: / for these bots (docs/ARCHITECTURE.md).
# ─────────────────────────────────────────────────────────────────────────────

# ── Traditional search + social unfurl ──────────────────────────────────────
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot
Allow: /
Disallow: /api/
Disallow: /version.json
Disallow: /cdn-cgi/

# ── AI-search citation crawlers ─────────────────────────────────────────────
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: DuckAssistBot
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /version.json
Disallow: /cdn-cgi/

# ── Training / bulk crawlers ────────────────────────────────────────────────
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: CCBot
User-agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /version.json
Disallow: /cdn-cgi/

# ── Everything else ─────────────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /api/
Disallow: /version.json
Disallow: /cdn-cgi/

# ─── Content Signals (draft-romm-aipref-contentsignals) ───
# Confirm this stance with Max before launch (SOW §2: declare ai-train, search, ai-input).
Content-Signal: ai-train=yes, search=yes, ai-input=yes

Sitemap: https://www.gtmnorth.ca/sitemap.xml

# Machine-readable site overview for AI agents.
llms.txt: https://www.gtmnorth.ca/llms.txt