# ============================================
# storeclaw.ai 爬虫策略
# AI 爬虫策略：欢迎引用，欢迎训练
# 复核日期：2026-05-12
# 复核频率：半年（按 darkvisitors.com 等公开注册表）
# ============================================

# ============================================
# 通用爬虫策略
# ============================================
User-agent: *
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug
Disallow: /helps/

# Block duplicate URLs with tracking/query params。

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

# ============================================
# AI 爬虫策略
# 每个 AI UA 块完整复制通用块的 Disallow 列表，
# 抵消 robots.txt"最具体 UA 块完全胜出"语义陷阱。
# 详见：openspec/changes/declare-ai-crawler-policy
# ============================================

# --- 训练型 ---

User-agent: GPTBot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: ClaudeBot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: anthropic-ai
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Google-Extended
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Applebot-Extended
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Amazonbot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Bytespider
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: CCBot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: cohere-ai
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Diffbot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: PerplexityBot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

# --- 答案 / 搜索型 ---

User-agent: OAI-SearchBot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Perplexity-User
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: DuckAssistBot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: YouBot
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

# --- 用户主动型 ---

User-agent: ChatGPT-User
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Claude-User
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Claude-Web
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /account/
Disallow: /zh/account/
Disallow: /bind/
Disallow: /zh/bind/
Disallow: /api/
Disallow: /app/api/
Disallow: /app/share-api/
Disallow: /auth/embed
Disallow: /zh/auth/embed
Disallow: /auth/desktop
Disallow: /zh/auth/desktop
Disallow: /crossborder-merchant
Disallow: /zh/crossborder-merchant
Disallow: /shopify-auth-complete
Disallow: /zh/shopify-auth-complete
Disallow: /payment-success
Disallow: /zh/payment-success
Disallow: /integration-auth-success
Disallow: /zh/integration-auth-success
Disallow: /integration-auth-failed
Disallow: /zh/integration-auth-failed
Disallow: /debug

Disallow: /*?utm_
Disallow: /*?back=
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?lang=

Sitemap: https://www.storeclaw.ai/sitemap.xml