# Artists in Canada — robots.txt
#
# Two policies live here, and they are deliberately different:
#
#   1. Ordinary search and AI SEARCH crawlers are welcome. Being findable — and being
#      citable when somebody asks an assistant about Canadian artists — is the point of
#      the directory, so those agents are listed explicitly rather than left to the
#      wildcard group, to make the intent unambiguous to anyone reading this file.
#
#   2. AI TRAINING crawlers and content scrapers are not. Members' artwork is not
#      training data.
#
# This file is the polite layer and is honoured only by agents that choose to. The
# enforcement layer is the Cloudflare "block AI trainers and scrapers, allow AI search"
# rule on this zone, which drops them at the edge whatever this file says. Keep the two
# in step: if the Cloudflare policy changes, change this file and the copy on
# /membership/hosting and /membership that describes it.
#
# Do NOT add Disallow rules for /cart/checkout or the other payment URLs. They carry
# X-Robots-Tag: noindex (App\Http\Middleware\NoIndex), and a URL blocked here is never
# recrawled — so Google would never see the noindex on the ones it has already indexed.
# See .ai/rules/sponsor-ads.md and tests/Feature/Seo/CheckoutNoIndexTest.php.

User-agent: *
Allow: /
Crawl-delay: 2

# ── AI search and user-initiated fetches: allowed ─────────────────────────────
# These answer a person's question and link back. They are not training crawlers.

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

# ── AI training and scraping: blocked ─────────────────────────────────────────
# Model training, dataset collection and bulk content harvesting.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: Scrapy
Disallow: /

Sitemap: https://artistsincanada.com/sitemap.xml