User-agent: *
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=no
Content-Usage: train-ai=n, search=y

# Answer-engine and search crawlers (AEO) that cite and link back are explicitly allowed;
# training-oriented crawlers (GPTBot, ClaudeBot, Google-Extended, Applebot-Extended,
# anthropic-ai and others below) are disallowed so this file, /terms/ and llms.txt agree.
# Search treatment is never restricted: Googlebot and Bingbot stay allowed.
# Answer-engine crawlers (AEO), explicitly allowed; /imprint/ is excluded from AI crawlers.
User-agent: OAI-SearchBot
Allow: /
Disallow: /imprint/

User-agent: PerplexityBot
Allow: /
Disallow: /imprint/

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Googlebot
Allow: /

User-agent: ChatGPT-User
Allow: /
Disallow: /imprint/

User-agent: Perplexity-User
Allow: /
Disallow: /imprint/

User-agent: Claude-SearchBot
Allow: /
Disallow: /imprint/

User-agent: Claude-User
Allow: /
Disallow: /imprint/

User-agent: Applebot-Extended
Disallow: /

User-agent: Bingbot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Aggressive SEO/marketing crawlers that add no AEO/citation value and
# only cost bandwidth + pollute logs, disallowed while the wildcard
# * rule and the named crawlers above stay untouched.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Policy: search and answer-engine crawlers that cite and link back are welcome;
# training-only and bulk-scraping crawlers are not (see /terms/).
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Further training-only and dataset crawlers. YouBot, DuckAssistBot and MistralAI-User
# are not listed on purpose: they cite and link back, so they fall under the wildcard group.
User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: FacebookBot
Disallow: /

Sitemap: https://www.weaponspecs.com/sitemap-index.xml
Sitemap: https://www.weaponspecs.com/news-sitemap.xml