# robots.txt — RFC 9309 compliant # https://www.rfc-editor.org/rfc/rfc9309 # General web crawlers (Googlebot, Bingbot, etc.) User-agent: * Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ Disallow: /*.json$ Disallow: /search? # AI crawlers — allowed, with private/sensitive paths blocked User-agent: GPTBot Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ User-agent: Google-Extended Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ User-agent: CCBot Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ User-agent: anthropic-ai Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ User-agent: Claude-Web Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ User-agent: cohere-ai Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ User-agent: PerplexityBot Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ User-agent: Bytespider Allow: / Disallow: /admin/ Disallow: /api/ Disallow: /private/ Disallow: /tmp/ Sitemap: https://www.simulationslabs.com/sitemap.xml