fix: correct baidu crawl configuration
This commit is contained in:
+23
-9
@@ -1,26 +1,40 @@
|
||||
# 对所有搜索引擎生效
|
||||
User-agent: *
|
||||
|
||||
# 允许抓取网站所有公开页面
|
||||
Allow: /
|
||||
|
||||
# 屏蔽登录/注册/用户中心页面
|
||||
# 登录、注册和用户中心
|
||||
Disallow: /login.html
|
||||
Disallow: /reg.html
|
||||
Disallow: /repwd.html
|
||||
Disallow: /usercenter.html
|
||||
Disallow: /html/login.html
|
||||
Disallow: /html/reg.html
|
||||
Disallow: /html/repwd.html
|
||||
Disallow: /html/usercenter.html
|
||||
|
||||
# 屏蔽财务相关页面
|
||||
# 财务页面
|
||||
Disallow: /chongzhi.html
|
||||
Disallow: /tixian.html
|
||||
Disallow: /tixianlist.html
|
||||
Disallow: /jine.html
|
||||
Disallow: /html/chongzhi.html
|
||||
Disallow: /html/tixian.html
|
||||
Disallow: /html/tixianlist.html
|
||||
Disallow: /html/jine.html
|
||||
|
||||
# 屏蔽收藏/分销/推广相关页面
|
||||
# 用户收藏、分销、推广和发布页面
|
||||
Disallow: /mianfeishoucang.html
|
||||
Disallow: /zhuanjiashoucang.html
|
||||
Disallow: /wodefenxiao.html
|
||||
Disallow: /fenxiao.html
|
||||
Disallow: /tuiguang.html
|
||||
Disallow: /fabumianfeiwenzhang.html
|
||||
Disallow: /fufei.html
|
||||
Disallow: /apply.html
|
||||
Disallow: /html/mianfeishoucang.html
|
||||
Disallow: /html/wodefenxiao.html
|
||||
Disallow: /html/fenxiao.html
|
||||
Disallow: /html/tuiguang.html
|
||||
Disallow: /html/fabumianfeiwenzhang.html
|
||||
Disallow: /html/fufei.html
|
||||
Disallow: /html/apply.html
|
||||
|
||||
# 提交 sitemap,搜索引擎根据 sitemap 快速抓取
|
||||
Sitemap: https://3d.3dyjs.cn/sitemap.xml
|
||||
Sitemap: https://www.scs8888.cn/sitemap.xml
|
||||
|
||||
Reference in New Issue
Block a user