# =====================================================
# 北京高端大圈品茶工作室 - robots.txt
# 网站: https://wangsen027.com
# 更新时间: 2026-06-06
# 说明: 优化百度等搜索引擎爬虫抓取策略
# =====================================================

# 允许所有搜索引擎爬虫访问
User-agent: *
Allow: /
Disallow: /css/
Disallow: /js/
Disallow: /images/
Disallow: /articles/
# 允许百度深度抓取文章和栏目页
Allow: /about/
Allow: /service/
Allow: /story/
Allow: /contact/
Allow: /index.html
Allow: /about/index.html
Allow: /service/index.html
Allow: /story/index.html
Allow: /contact/index.html

# 百度爬虫专项优化
User-agent: Baiduspider
Allow: /
Allow: /about/
Allow: /service/
Allow: /story/
Allow: /contact/
Allow: /index.html
Crawl-delay: 1

# 百度移动端爬虫
User-agent: Baiduspider-mobile
Allow: /
Allow: /about/
Allow: /service/
Allow: /story/
Allow: /contact/

# 360搜索爬虫
User-agent: 360Spider
Allow: /
Allow: /about/
Allow: /service/
Allow: /story/
Allow: /contact/

# 搜狗爬虫
User-agent: Sogou web spider
Allow: /
Allow: /about/
Allow: /service/
Allow: /story/
Allow: /contact/

# 神马搜索爬虫
User-agent: SemrushBot
Allow: /
Allow: /about/
Allow: /service/
Allow: /story/
Allow: /contact/

# 禁止恶意爬虫
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemap 位置
Sitemap: https://wangsen027.com/sitemap.xml