# BhoomiScan Robots.txt
# Last updated: May 2026 (Phase 18 SEO upgrade)

# Sitemap index and sub-sitemaps
Sitemap: https://www.bhoomiscan.in/sitemap.xml
Sitemap: https://www.bhoomiscan.in/sitemap-pages.xml
Sitemap: https://www.bhoomiscan.in/sitemap-blog.xml
Sitemap: https://www.bhoomiscan.in/sitemap-locations.xml
Sitemap: https://www.bhoomiscan.in/sitemap-glossary.xml
Sitemap: https://www.bhoomiscan.in/sitemap-news.xml
Sitemap: https://www.bhoomiscan.in/sitemap-authors.xml
Sitemap: https://www.bhoomiscan.in/sitemap-faq.xml
Sitemap: https://www.bhoomiscan.in/sitemap-name-search.xml
Sitemap: https://www.bhoomiscan.in/sitemap-district-hubs.xml
Sitemap: https://www.bhoomiscan.in/sitemap-pillars.xml
Sitemap: https://www.bhoomiscan.in/sitemap-advocates.xml
Sitemap: https://www.bhoomiscan.in/sitemap-odia.xml

# RSS / Atom feeds (also referenced via <link rel="alternate">)
# https://www.bhoomiscan.in/rss.xml
# https://www.bhoomiscan.in/feed.xml

# AI models — llms.txt is the structured AI hint index, llms-full.txt is
# the full knowledge bundle (pillars + glossary + recent articles
# concatenated). These are NOT XML sitemaps — they shouldn't be declared
# as `Sitemap:` lines (Google rejects non-XML targets). Both surfaces are
# live and discoverable via on-page <link rel="alternate"> hints + UA-aware
# crawler routing in vercel.json.
# /llms.txt
# /llms-full.txt

# Default rules for all crawlers
User-agent: *
Allow: /

# NOTE ON /go/wa (the tracked contact redirect): it is deliberately NOT
# Disallow-ed here. Two reasons, both learned the hard way.
#
# 1. A Disallow in this `User-agent: *` group would not bind the crawlers it
#    names. Under RFC 9309 a crawler obeys exactly ONE group — its most
#    specific match — and Googlebot, Bingbot, GPTBot, ClaudeBot, PerplexityBot,
#    CCBot and the rest each have their own group further down containing only
#    `Allow: /`. They never read this group at all. (That also means the
#    Disallow lines below have never bound those crawlers either — a
#    pre-existing issue, flagged rather than silently changed here.)
# 2. Blocking a URL that is linked from every blog post would make the
#    handler's own `X-Robots-Tag: noindex` UNREADABLE — Google cannot fetch a
#    header it is forbidden to request — which is the classic recipe for
#    "Indexed, though blocked by robots.txt".
#
# The URL is instead kept out of the index by `rel="nofollow"` on every anchor
# plus the `X-Robots-Tag: noindex, nofollow` the redirect actually serves.

# Block protected/authenticated routes
Disallow: /admin
Disallow: /admin-debug
Disallow: /admin/
Disallow: /v2/admin
Disallow: /v2/admin/
Disallow: /scans
Disallow: /new-scan
Disallow: /report/
Disallow: /v2/login
Disallow: /dashboard
Disallow: /profile
Disallow: /list-property
Disallow: /list-property-meta
Disallow: /favorites
Disallow: /test-funnel-tracking
Disallow: /v2/ror-test

# Block auth callbacks (redirect targets, never indexable)
Disallow: /auth/

# Block API and utility paths
Disallow: /api/
Disallow: /_/

# Block /properties?... filter URLs flagged as Soft 404 by Google
# (city/type filter combinations where most have zero listings behind them
# and city values were polluted with village names). Bare /properties stays
# crawlable. /land/odisha/* is also crawlable — the sitemap now emits only
# geography-validated URLs from v_sitemap_locations.
Disallow: /properties?*

# Allow social media bots for rich previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# Google-specific rules
User-agent: Googlebot
Allow: /

# Bing-specific rules
User-agent: Bingbot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl (used by AI training datasets)
User-agent: CCBot
Allow: /

# You.com
User-agent: YouBot
Allow: /