# RoboHorizon - Global Robotics and Automation Magazine
# Last updated: 2026-05-30

# Major search engines and AI crawlers: full access.
# Tag pages have <meta name="robots" content="noindex, follow"> in HTML —
# Google needs to crawl them to see the noindex directive and deindex them.
# Blocking via robots.txt prevents Google from ever confirming noindex.
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: facebookexternalhit
User-agent: GPTBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: CCBot
User-agent: cohere-ai
User-agent: PerplexityBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: Sogou
User-agent: Exabot
User-agent: facebot
User-agent: ia_archiver
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: PetalBot
User-agent: seznambot
User-agent: magpie-crawler
User-agent: Bytespider
Allow: /

# Default rule for all other bots
User-agent: *
Disallow: /admin/
Disallow: /system/
Disallow: /tmp/
Disallow: /cache/
Disallow: /search/
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /404.html
Disallow: /print/
Disallow: /preview/
Disallow: /*/draft-*
Disallow: /*?*query=
Disallow: /*?*s=
Disallow: /*?*search=
Disallow: /*?*page=
Disallow: /*?*lang=
Disallow: /feed.xml$
Disallow: /rss.xml$
Disallow: /atom.xml$
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /

# Sitemaps — main indexes per domain
Sitemap: https://robohorizon.com/sitemap.xml
Sitemap: https://robohorizon.uk/sitemap.xml
Sitemap: https://robohorizon.de/sitemap.xml
Sitemap: https://robohorizon.hu/sitemap.xml
Sitemap: https://robohorizon.eu/sitemap.xml
Sitemap: https://robohorizon.cn/sitemap.xml

# Language-specific sitemaps
Sitemap: https://robohorizon.com/en-us/sitemap.xml
Sitemap: https://robohorizon.com/zh-hant/sitemap.xml
Sitemap: https://robohorizon.com/ja/sitemap.xml
Sitemap: https://robohorizon.com/ko/sitemap.xml
Sitemap: https://robohorizon.uk/en-gb/sitemap.xml
Sitemap: https://robohorizon.de/de/sitemap.xml
Sitemap: https://robohorizon.hu/hu/sitemap.xml
Sitemap: https://robohorizon.eu/fr/sitemap.xml
Sitemap: https://robohorizon.eu/it/sitemap.xml
Sitemap: https://robohorizon.eu/pl/sitemap.xml
Sitemap: https://robohorizon.eu/nl/sitemap.xml
Sitemap: https://robohorizon.eu/es/sitemap.xml
Sitemap: https://robohorizon.eu/pt/sitemap.xml
Sitemap: https://robohorizon.cn/zh/sitemap.xml