User-agent: Google-Extended
Allow: /

# Search / assistant agents (Googlebot, bingbot, Baiduspider, PetalBot, Applebot, OAI-SearchBot,
# ChatGPT-User, Claude-SearchBot, Claude-User, PerplexityBot) are allowed so pages can be cited.
# Baiduspider has its own group near the end: the same rules as *, plus /api/ and the
# /schools/school filter URLs.
# Bytespider (ByteDance) is allowed on purpose: Doubao's page fetcher identifies as
# Bytespider and refuses to read the site while it is disallowed here (2026-09-21).
# KimiBot (Moonshot) is allowed for the same reason.
User-agent: Bytespider
Allow: /
Disallow: /trap

User-agent: KimiBot
Allow: /
Disallow: /trap

# AI training crawlers. Cloudflare's "Manage robots.txt" used to inject this list at the edge,
# but its version also disallowed Baiduspider / PetalBot / Google-Extended / KimiBot, so the
# list is kept here and the Cloudflare-managed block is switched off (2026-09-22).
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: MistralAI-Training
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: AwarioRssBot
Disallow: /

User-agent: QualifiedBot
Disallow: /

User-agent: Cotoyogi
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: atlassian-bot
Disallow: /

User-agent: FishBot
Disallow: /

User-agent: BorderxBot
Disallow: /

User-agent: NavuBot
Disallow: /

User-agent: WARDBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: CitibotSiteCrawler
Disallow: /

# Semrush 的工具爬虫：给我们自己的站点跑 Site Audit、On Page SEO Checker、Backlink Audit、写作助手等
# 工具时用，2026-09-29 放开（UA 名单见 https://www.semrush.com/bot/）。规则与下面的 User-agent: * 组相同。
# - 这一组必须写在下面的 User-agent: SemrushBot 之前：SemrushBot-SI 会把 SemrushBot 组当成管自己的
#   （按前缀匹配、先匹配上的组生效），放开前它只读 robots.txt、一个页面都不抓。
# - Allow: / 放在组尾，是给把 SemrushBot 组和这一组合并判定的解析器看的；放在最后不影响按
#   「第一条匹配规则」判定的解析器先看到上面的 Disallow。
User-agent: SiteAuditBot
User-agent: SemrushBot-SI
User-agent: SemrushBot-BA
User-agent: SemrushBot-SWA
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-FT
User-agent: SemrushBot-ESI
User-agent: SplitSignalBot
Disallow: /trap
Disallow: /trap/honey-ppt
Disallow: /*?rt=
Disallow: /*&rt=
Disallow: /labour-agreements?*occupationAnzsco=*&page=
Disallow: /en-US/labour-agreements?*occupationAnzsco=*&page=
Disallow: /zh-TW/labour-agreements?*occupationAnzsco=*&page=
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# Block common SEO crawlers that consume resources (optional, remove if you need SEO tools like Ahrefs)
User-agent: AhrefsBot
Disallow: /

# 只挡 Semrush 建外链库的全站爬虫；它给我们自己用的工具爬虫在上面单独放开
User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

Sitemap: https://www.oneuedu.com/sitemap.xml

# 百度单独一组：Baiduspider 抓 HTML，Baiduspider-render 是渲染页面时去取 JS / 接口的那个。
# - 有了专属组，百度只认这一组、不再看下面的 User-agent: *，所以 * 组的规则在这里全部再写一遍
#   （尤其是 /trap），改 * 组时两边一起改。
# - 百度按「第一条匹配上的 Allow / Disallow」判定（谷歌按最长匹配），例外的 Allow 必须写在对应的
#   Disallow 前面；也别加 Allow: /，它排在前面的话后面的 Disallow 全部失效。组内不留空行。
# 比 * 组只多挡两块（09-27/28 百度回归后的两天）：
# - /api/：渲染器约 1.2 万次请求 /api/id（版本号接口，与正文无关）。职业详情页「劳工协议」面板
#   在浏览器里取 /api/labour-agreements/types，渲染要用，单独放行。
# - /schools/school? 的筛选 / 翻页组合：约 2 万次，全是未缓存的 SSR；这些 URL 的 canonical
#   都指向不带参数的 /schools/school，学校详情页由 sitemap 收录。不带参数的 /schools/school 照常抓。
User-agent: Baiduspider
User-agent: Baiduspider-render
Allow: /api/labour-agreements/types
Disallow: /api/
Disallow: /schools/school?
Disallow: /en-US/schools/school?
Disallow: /zh-TW/schools/school?
Disallow: /trap
Disallow: /trap/honey-ppt
Disallow: /*?rt=
Disallow: /*&rt=
Disallow: /labour-agreements?*occupationAnzsco=*&page=
Disallow: /en-US/labour-agreements?*occupationAnzsco=*&page=
Disallow: /zh-TW/labour-agreements?*occupationAnzsco=*&page=
Content-Signal: ai-train=no, search=yes, ai-input=yes

# 改这一组时，同步改上面的百度组和 Semrush 工具组
User-agent: *
Disallow: /trap
Disallow: /trap/honey-ppt
# 旧版 AppUpdater 自动重载留下的 ?rt=N&t=<id> 变体，正文与裸 URL 相同，不必抓取
Disallow: /*?rt=
Disallow: /*&rt=
# 劳工协议按职业筛选的分页（?occupationAnzsco=<code>&page=N）：1,252 个职业 × 最多 173 页的
# getServerSideProps 变体，canonical 全指向裸列表、没有收录价值，Googlebot 三天抓了 2.7 万次。
# 只挡带 page= 的翻页；职业筛选首页（不带 page）保留可抓，页面本身已 noindex,follow。
Disallow: /labour-agreements?*occupationAnzsco=*&page=
Disallow: /en-US/labour-agreements?*occupationAnzsco=*&page=
Disallow: /zh-TW/labour-agreements?*occupationAnzsco=*&page=
Content-Signal: ai-train=no, search=yes, ai-input=yes