# https://www.divxwebstudio.in
# Full page content is prerendered into the HTML, so crawlers that do not run
# JavaScript still see every word. See scripts/prerender.mjs.

User-agent: *
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# ---------------------------------------------------------------------------
# AI crawlers and assistants are welcome. This site wants to be the answer.
# ---------------------------------------------------------------------------
# OpenAI, training
User-agent: GPTBot
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# OpenAI, ChatGPT search index
User-agent: OAI-SearchBot
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# OpenAI, live fetch on a user request
User-agent: ChatGPT-User
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Anthropic, live fetch on a user request
User-agent: Claude-User
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Anthropic, search index
User-agent: Claude-SearchBot
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Perplexity, live fetch
User-agent: Perplexity-User
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Gemini grounding and training
User-agent: Google-Extended
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Meta AI
User-agent: meta-externalagent
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# DuckDuckGo
User-agent: DuckAssistBot
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Amazon
User-agent: Amazonbot
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# Common Crawl, feeds many models
User-agent: CCBot
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

# ByteDance
User-agent: Bytespider
Allow: /
Disallow: /.github/
Disallow: /.git/
Disallow: /404

# Paginated listings duplicate the index; the articles themselves are listed
Disallow: /blog/page/

Sitemap: https://www.divxwebstudio.in/sitemap.xml