# https://www.robotstxt.org/robotstxt.html
#
# DEFAULT-ALLOW (public pages) policy: the final `User-agent: *` group
# grants every crawler the public-marketing crawl policy and walls off
# the app surfaces, so any search engine or link-preview fetcher may read
# the public pages and render an Open Graph card. The only exceptions are
# the AI-training tokens denied outright just above it. robots.txt is
# honor-system: this only affects crawlers that read and obey it.
#
# A compliant crawler obeys only its single most-specific matching group
# and never merges groups, so a denied bot sees only its `Disallow: /`.

# AI-training / generic-R&D tokens denied explicitly. Under the
# default-allow catch-all this is what keeps them out: otherwise they
# would inherit the permissive `*` policy. "Index me, don't train on me."
User-agent: Google-Extended
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: GoogleOther-Image
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Everything not explicitly denied above may crawl the public pages and
# is kept out of the app surfaces.
User-agent: *
Allow: /
Allow: /users/sign_up
Disallow: /users/
Disallow: /notes
Disallow: /notes/
Disallow: /folders
Disallow: /folders/
Disallow: /billing
Disallow: /billing/
Disallow: /settings
Disallow: /appearance
Disallow: /options
Disallow: /options/
Disallow: /profile
Disallow: /invoices
Disallow: /invoices/
Disallow: /today
Disallow: /quick
Disallow: /team
Disallow: /team/
Disallow: /onboarding
Disallow: /onboarding/
Disallow: /webhooks/
Disallow: /api/
Disallow: /mcp
Disallow: /h/
Disallow: /hb/
Disallow: /up
Disallow: /manifest
Disallow: /service-worker
Disallow: /n/
Disallow: /trap

Sitemap: https://freshjots.com/sitemap.xml