# ===================================================================
# 🛡️ PROTECTED ARCHITECTURE - AELION DIGITAL AGENCY
# Location: London Headquarters (Great James Street, WC1N)
# Compliance: RFC 9309 Standard (UTF-8) | Status: Production 2026
# Ref ID: OPS-SRE-2026-DIAMOND-X72-V5.1
#
# NOTICE: This configuration is proprietary intellectual property.
# Unauthorised mirroring is detected via edge metadata signatures.
# ===================================================================

# [SECTION 1] ORGANIC DISCOVERABILITY
# Primary search indexers and inspection tools.
User-agent: Googlebot
User-agent: Google-InspectionTool
User-agent: GoogleOther
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot

# [SECTION 2] GENERATIVE ENGINE OPTIMISATION (GEO)
# Citation-based and search-indexing agents.
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: MistralAI-Index
User-agent: DuckAssistBot
User-agent: Meta-WebIndexer

# [SECTION 2B] USER-REQUESTED FETCHERS
# User-initiated fetchers that may retrieve content on demand.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Google-Agent
User-agent: Google-NotebookLM
User-agent: Meta-ExternalFetcher
User-agent: facebookexternalhit

# Shared access policy for approved indexers and fetchers
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /wp-config.php
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /readme.html

Disallow: /?s=
Disallow: /search/
Disallow: /*?sort=
Disallow: /*?orderby=
Disallow: /*?utm_*
Disallow: /*?ref=
Disallow: /*add-to-cart=*

Allow: /articles-library/*.pdf$
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /*.rar$

Allow: /wp-admin/admin-ajax.php
Allow: /favicon.ico
Allow: /*.js
Allow: /*.css

Crawl-delay: 2

# [SECTION 3] NON-REFERRAL SCRAPERS, MODEL TRAINERS & ENTERPRISE INGESTION
# Blocked to prevent training, third-party ingestion, or non-search harvesting.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: amazon-QBusiness
User-agent: amazon-kendra
User-agent: CCBot
User-agent: Diffbot
User-agent: FirecrawlAgent
Disallow: /

# [SECTION 4] COMMERCIAL AUDIT & COMPETITIVE INTELLIGENCE
# Access restricted to prevent automated commercial reconnaissance.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: Rogerbot
User-agent: DotBot
User-agent: AhrefsSiteAudit
User-agent: SiteAuditBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SplitSignalBot
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-FT
User-agent: SemrushBot-ESI
User-agent: RyteBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: SeobilityBot
User-agent: Seobility
User-agent: SiteCheckerBotCrawler
User-agent: serpstatbot
User-agent: SerpstatBot
User-agent: Screaming Frog SEO Spider
User-agent: Oncrawl
User-agent: OncrawlBot
User-agent: SISTRIX
User-agent: SISTRIX Crawler
User-agent: sistrix
User-agent: Barkrowler
User-agent: SEOkicks-Robot
User-agent: SEOkicks
User-agent: MegaIndex.ru
User-agent: megaindex.com
User-agent: SearchmetricsBot
User-agent: Lipperhey
User-agent: Lipperhey-Kaus-Australis
User-agent: BacklinkCrawler
User-agent: spbot
User-agent: SEOdiver
User-agent: cocolyzebot
User-agent: AwarioBot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
Disallow: /

# [SECTION 5] REGIONAL TRAFFIC GOVERNANCE
# Filtering non-target regional crawlers to optimise server P99 latency.
User-agent: Yandex
User-agent: Baiduspider
User-agent: Sogou web spider
Disallow: /

# [SECTION 6] DEFAULT SYSTEM HARDENING & PATH OBFUSCATION
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /wp-config.php
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /readme.html

Disallow: /?s=
Disallow: /search/
Disallow: /*?sort=
Disallow: /*?orderby=
Disallow: /*?utm_*
Disallow: /*?ref=
Disallow: /*add-to-cart=*

Allow: /articles-library/*.pdf$
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /*.rar$

Allow: /wp-admin/admin-ajax.php
Allow: /favicon.ico
Allow: /*.js
Allow: /*.css

# -------------------------------------------------------------------
# ARCHITECTURAL NOTES:
# 1. Mirrored at Edge WAF (Cloudflare/AWS AI Crawl Control).
# 2. Automated null-routing for blocked agents.
# 3. AI-friendly Markdown index located at /llms.txt & /llms-full.txt
# 4. Supports real-time indexing via IndexNow protocol.
# 5. SRE Recruitment: aelion.ae/careers
# -------------------------------------------------------------------
Sitemap: https://www.aelion.ae/sitemap_index.xml