# --- SEO scrapers / backlink tools: block entire site ---
User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SERankingBacklinksBot
Disallow: /

User-agent: Bytespider
Disallow: /


# --- AI training bots ---
User-agent: GPTBot
Disallow: /api/

User-agent: ChatGPT-User
Disallow: /api/

User-agent: Google-Extended
Disallow: /api/

User-agent: ClaudeBot
Disallow: /api/

User-agent: anthropic-ai
Disallow: /api/

User-agent: PerplexityBot
Disallow: /api/

User-agent: CCBot
Disallow: /api/

User-agent: Amazonbot
Disallow: /api/


# --- Social link preview crawlers: allow sharing previews, block APIs ---
User-agent: facebookexternalhit
Allow: /jobs$
Allow: /company/
Allow: /og/
Disallow: /api/

User-agent: Facebot
Allow: /jobs$
Allow: /company/
Allow: /og/
Disallow: /api/

User-agent: LinkedInBot
Allow: /jobs$
Allow: /company/
Allow: /og/
Disallow: /api/

User-agent: Twitterbot
Allow: /jobs$
Allow: /company/
Allow: /og/
Disallow: /api/

User-agent: XBot
Allow: /jobs$
Allow: /company/
Allow: /og/
Disallow: /api/


# --- Search Engines & Everyone else ---
User-agent: *

# Allow core crawlable surfaces.
Allow: /jobs$
Allow: /company/
Allow: /og/

# Block company/query duplicates such as ?prev=.
Disallow: /*?*prev=

# ------------------------------------------------------------
# /jobs crawl policy
#
# Allowed clean forms only:
# /jobs?jobCategories=...&locations=...
# /jobs?locations=...&jobCategories=...
#
# Everything else under /jobs with query params is blocked.
# This blocks category-only, location-only, tech, level, salary,
# pagination, sorting, search, remote, and all mixed extra-param URLs.
# ------------------------------------------------------------

# Allow only the two clean category + location query patterns.
Allow: /jobs?jobCategories=*&locations=*$
Allow: /jobs?locations=*&jobCategories=*$

# Block extra param BEFORE clean category/location.
Disallow: /jobs?*&jobCategories=*&locations=*
Disallow: /jobs?*&locations=*&jobCategories=*

# Block extra param BETWEEN category and location.
Disallow: /jobs?jobCategories=*&*&locations=*
Disallow: /jobs?locations=*&*&jobCategories=*

# Block extra param AFTER clean category/location.
Disallow: /jobs?jobCategories=*&locations=*&*
Disallow: /jobs?locations=*&jobCategories=*&*

# Explicitly block known faceted/internal params anywhere under /jobs.
Disallow: /jobs?*includedTech=
Disallow: /jobs?*excludedTech=
Disallow: /jobs?*experienceLevel=
Disallow: /jobs?*minSalary=
Disallow: /jobs?*maxSalary=
Disallow: /jobs?*salary=
Disallow: /jobs?*locationType=
Disallow: /jobs?*remote=
Disallow: /jobs?*page=
Disallow: /jobs?*perPage=
Disallow: /jobs?*sort=
Disallow: /jobs?*aiApplyOnly=
Disallow: /jobs?*zoneOnly=
Disallow: /jobs?*q=
Disallow: /jobs?*query=
Disallow: /jobs?*search=
Disallow: /jobs?*tech=

# Block all other /jobs query URLs, including:
# /jobs?jobCategories=...
# /jobs?locations=...
# /jobs?page=...
# /jobs?salary=...
# /jobs?experienceLevel=...
# /jobs?includedTech=...
# /jobs?anythingElse=...
Disallow: /jobs?

# Also block slash-query variants, just in case they exist.
Disallow: /jobs/?

# Block APIs and legacy API aliases.
Disallow: /api/
Disallow: /SlashAPI
Disallow: /SlashAPI/


# --- Sitemaps ---
Sitemap: https://jobtailor.co/sitemaps/sitemap-index.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-0.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-1.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-2.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-3.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-4.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-5.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-6.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-7.xml
Sitemap: https://jobtailor.co/sitemaps/job-postings-companies.xml
Sitemap: https://jobtailor.co/sitemaps/jobtitle-location.xml