# hgs.com robots.txt
# Rewritten 2026-06. Policy: this file controls crawling of site
# infrastructure only. Indexing of individual pages is controlled
# on-page (meta robots or X-Robots-Tag header), never here.
# Do not add content URLs to this file.

# --- AI crawler policy ---
# Decision: HGS blocks AI training/grounding crawlers on all owned
# domains. Deploy this same block list and the ai-train=no content
# signal on joinhgs.com.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

# --- All other crawlers ---
User-agent: *
# WordPress system paths
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-content/js/
Disallow: /wp-content/plugins/
Disallow: /wp-content/uploads/wpforms/
# Template fragments rendered as standalone URLs
Disallow: /footer/
Disallow: /footer-2024/
Disallow: /solution-pre-footer/
Disallow: /footer-insight-pages/
# Thin taxonomy pages
Disallow: /tag/
# Form completion pages (generic patterns, no campaign names)
Disallow: /confirmation/
Disallow: /*-thank-you/
Crawl-delay: 10

Sitemap: https://hgs.com/sitemap_index.xml