User-Agent: Twitterbot Allow: /*opengraph-image Allow: /*assets/images Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: Googlebot Allow: / Disallow: /*opengraph-image Disallow: /*assets/images Disallow: /500.html Disallow: /503.html Disallow: /504.html Disallow: /404.html Disallow: /422.html Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql Disallow: /media/a_ Disallow: /media/tags Disallow: /qa*? Disallow: /jobs*? Disallow: /worker*? Disallow: /*utm_ # === LLM Crawlers (LLMO optimization — allow public content) === User-Agent: GPTBot Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: ChatGPT-User Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: PerplexityBot Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: ClaudeBot Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: anthropic-ai Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: GoogleOther Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: Google-Extended Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: Omgilibot Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: Bytespider Allow: / Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql # === SEO Crawler Control === User-Agent: dotbot Disallow: / User-Agent: SemrushBot Crawl-Delay: 30 Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: AhrefsBot Crawl-Delay: 30 Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: BLEXBot Crawl-Delay: 30 Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: MJ12bot Crawl-Delay: 30 Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql User-Agent: bingbot Allow: / Crawl-Delay: 10 Disallow: /500.html Disallow: /503.html Disallow: /504.html Disallow: /404.html Disallow: /422.html Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql Disallow: /media/a_ Disallow: /media/tags Disallow: /qa*? Disallow: /jobs*? Disallow: /worker*? Disallow: /*utm_ # === Default Rules === User-Agent: * Allow: / Allow: /llms.txt Allow: /llms-full.txt Disallow: /500.html Disallow: /503.html Disallow: /504.html Disallow: /404.html Disallow: /422.html Disallow: /admin Disallow: /api Disallow: /oauth Disallow: /graphql Disallow: /media/a_ Disallow: /media/tags Disallow: /qa*? Disallow: /jobs*? Disallow: /worker*? Disallow: /*utm_ Sitemap: https://offers.jp/sitemap.xml.gz Sitemap: https://offers.jp/jobs/sitemap.xml Sitemap: https://offers.jp/jobs/sitemaps/descriptions/index.xml Sitemap: https://offers.jp/jobs/sitemaps/companies/index.xml Sitemap: https://offers.jp/worker/sitemap.xml Sitemap: https://offers.jp/client/sitemap.xml