# robots.txt — technovids.com
#
# Search engines and AI answer engines are welcome to crawl the public site.
# AI crawlers are named explicitly (SEO audit gap #24, 2 October 2026) so the
# policy is a deliberate choice, not a default: Technovids wants to be found,
# quoted and cited accurately by AI assistants, so both the crawlers that
# fetch pages for live answers and those that collect training data are
# allowed. To opt out of model training only, change "Allow: /" to
# "Disallow: /" in the training group below; answer engines keep access.
#
# Every group repeats the private paths, because a crawler that matches a
# named group ignores the "*" group:
#   /design-system  internal page (also noindex and excluded from the sitemap)
#   /api/           form endpoints, not content

# --- AI answer and search crawlers (fetch pages to answer and cite) ----------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
Allow: /
Disallow: /design-system
Disallow: /api/

# --- AI training crawlers (help models know Technovids accurately) ----------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
Allow: /
Disallow: /design-system
Disallow: /api/

# --- Everyone else, including Googlebot and Bingbot -------------------------
User-agent: *
Allow: /
Disallow: /design-system
Disallow: /api/

Sitemap: https://technovids.com/sitemap-index.xml

# Summary for AI assistants and answer engines (llmstxt.org):
#   https://technovids.com/llms.txt       short index
#   https://technovids.com/llms-full.txt  every page's answer and FAQs