# Content Signals (https://contentsignals.org/) - usage preferences, not access control.
#   search=yes    - index and link to us (the SEO bet)
#   ai-input=yes  - answer engines may ground answers in our pages and cite us (SEO-02)
#   ai-train=no   - titles/summaries are regulators' own words, much of it non-commercial
#                   or permission-only (docs/legal/SOURCE_LICENSING.md); not ours to offer
# Repeated per group: a crawler obeys only its most specific User-agent group.
#
# Disallow /updates/finra- (every group): FINRA's terms prohibit scraping and
# restrict content to non-commercial use. Withdrawn Aug 2026; the 108 existing
# records stay published but are not offered to crawlers (SOURCE_LICENSING.md).
# Longest match wins, so this overrides the broader Allow: /.

# AI Search Crawlers - Explicitly Allow
User-agent: GPTBot
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: OAI-SearchBot
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: ChatGPT-User
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: PerplexityBot
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: ClaudeBot
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: anthropic-ai
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no

# Traditional Search Crawlers
User-agent: Googlebot
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: Bingbot
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no

# Default - Allow All
User-agent: *
Allow: /
Disallow: /updates/finra-
Content-Signal: search=yes, ai-input=yes, ai-train=no


Sitemap: https://regmonitoring.com/sitemap.xml