# robots.txt for https://www.sisystems.site/
# Policy: this site is intentionally public and AI-discoverable.
# robots.txt is advisory only. It does not force any crawler or agent to visit,
# and it is not access control. Nothing private is served on this domain.
#
# Private areas: this is a static marketing site with no admin, account, login,
# checkout or private API paths. If any are added later, list them here as
# Disallow lines under User-agent: * (for example: Disallow: /admin/).

# Public web access
User-agent: *
Allow: /

# ---------------- OpenAI ----------------
# Training crawler. Allowed by owner choice; not required to appear in ChatGPT Search.
User-agent: GPTBot
Allow: /

# ChatGPT Search discovery
User-agent: OAI-SearchBot
Allow: /

# User-requested ChatGPT fetches
User-agent: ChatGPT-User
Allow: /

# ---------------- Anthropic ----------------
# Training crawler. Allowed by owner choice.
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# ---------------- Google ----------------
User-agent: Googlebot
Allow: /

# Gemini extended model use. Allowed by owner choice.
User-agent: Google-Extended
Allow: /

# ---------------- Microsoft / Bing ----------------
# Also the search path used by Microsoft Copilot and GitHub Copilot web search.
User-agent: Bingbot
Allow: /

# ---------------- Perplexity ----------------
User-agent: PerplexityBot
Allow: /

# ---------------- Apple ----------------
User-agent: Applebot
Allow: /

# Extended AI/model-training use. Allowed by owner choice.
User-agent: Applebot-Extended
Allow: /

# ---------------- ByteDance ----------------
User-agent: Bytespider
Allow: /

# ---------------- Cohere ----------------
User-agent: cohere-ai
Allow: /

# ---------------- AI2 ----------------
User-agent: AI2Bot
Allow: /

# ---------------- Cursor ----------------
User-agent: Cursor
Allow: /

# ---------------- Zapier (Central, Agents, AI by Zapier) ----------------
User-agent: Zapier
Allow: /

# ---------------- xAI / Grok ----------------
# Reported compatibility tokens only. These are NOT official xAI crawler
# declarations. Grok may fetch through browsers or search infrastructure, which
# the User-agent: * rule above already permits.
User-agent: GrokBot
Allow: /

User-agent: xAI-SearchBot
Allow: /

User-agent: xAI-Bot
Allow: /

# ---------------- Agents with no published crawler token ----------------
# Devin, GitHub Copilot, OpenClaw, Hermes Agent, Hark, LangChain, LangGraph,
# CrewAI, AutoGen, Glean, n8n, B44 and Superagents publish no dedicated
# robots.txt token, so none is invented here. They fetch through normal
# browsers, search engines (Bing, Google) or configurable providers (Firecrawl,
# Exa, Tavily, Brave, SearXNG and others), all permitted by User-agent: * above.

Sitemap: https://www.sisystems.site/sitemap.xml