# Updated 2025-Dec-19 @ 13.20

# Default configuration

User-agent: *

# NOTE: Google never honours the crawl-delay directive, but other bots do
Crawl-Delay: 3

# NOTE: Relatively, new, porposed by CloudFlare, therefore unlikely to be honoured
# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.

# Content-signal: search=yes,ai-train=no


# Put customer specific allows here
# NOTE: Sitemap directives should be very bottom of file [if not being auto-generated by Magento]

# DELETE AS REQUIRED: Some sites may require these, function not totally clear
Allow: /catalog/product_compare/
Allow: /catalog/category/view/
Allow: /catalog/product/view/


# Disable customer specific items


# Disable system related directories
#
Disallow: /404/
Disallow: /admin/
Disallow: /ajax/
Disallow: /api/
Disallow: /app/
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /dev/
Disallow: /downloader/
Disallow: /generated/
Disallow: /includes/
Disallow: /index.php/
Disallow: /lib/
Disallow: /log/
Disallow: /magento/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /setup/
Disallow: /staging/
Disallow: /stats/
Disallow: /tmp/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/
Disallow: /SiteAdmin/

# Disable standard search pages
#
Disallow: /catalogsearch/
Disallow: /instantsearch/
Disallow: /search/

# Disable 3rd-party search modules
#
Disallow: /eccsearch/
Disallow: /mageworx_searchsuiteautocomplete/
Disallow: /mfproductsearch/

# Disable checkout & customer account
#
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /b2b/
Disallow: /customer/


# Ensure only main, clean URLs are indexed by search engines 
#Disallow: /*?

# Need to allow PLP pages	/apparel.html?p=2
# But block things like		/apparel.html?p=2&this=that
#							/catalogsearch/result/index/?ajax_nav=1&brand=5512&p=2&q=air+compressor
Allow: /*?p=
#Disallow: /*?p=*&

#Some customer may want to allow categories for Google
#Allow: /*cat=

# Need to allow specific Google and Bing et al related marketing parameters
Allow: /*?utm
Allow: /*?gad
Allow: /*?gclid
Allow: /*?dclid
Allow: /*?srsltid
Allow: /*?msclkid
Allow: /*?_gl



# Blocking Nuisance Bots

# Bots you may or may not want


# This bot may require being a single statement
User-Agent: SemrushBot
User-Agent: SemrushBot-BA
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: SemrushBot-SI
User-agent: SplitSignalBot
User-agent: SiteAuditBot
Allow: /
Disallow: /customer/
Disallow: /rest/
Crawl-Delay: 5



# Control over whether content is used to train Google's generative AI models
User-agent: Google-Extended

# Used by Google Internal for R&D, separate to GoogleBot
# Not tied to: Google Search or ranking.
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video

# Bots related to OpenAI
User-Agent: GPTBot
User-Agent: ChatGPT-User
User-Agent: OAI-SearchBot
User-Agent: TerraCotta

# Bots related to Amazon & Facebook
User-Agent: Amazonbot
User-Agent: facebookexternalhit
User-Agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: meta-webindexer

User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-Agent: aiHitBot
User-Agent: AhrefsBot

# Alibaba Cloud (Aliyun), specifically for security and monitoring purposes. [Hong Kong / China]
User-Agent: AliyunSecBot
User-agent: anthropic-ai
User-Agent: Applebot
User-agent: Applebot-Extended
User-agent: AspiegelBot
User-agent: BacklinksExtendedBot
User-agent: Baiduspider
User-Agent: Barkrowler
User-Agent: BitSightBot
User-Agent: Brightbot
User-Agent: Bytespider
User-agent: CCBot
User-Agent: CheckMarkNetwork
User-agent: Claude-Web
User-agent: ClaudeBot
User-Agent: Cliqzbot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Crawlspace
User-Agent: coccocbot-image
User-Agent: coccocbot-web
User-Agent: DataProvider
User-Agent: DataForSeoBot
User-agent: Diffbot
User-Agent: Discordbot
User-Agent: DomainStatsBot
User-Agent: Dormouse
User-Agent: DotBot
User-agent: DuckAssistBot
User-Agent: DuckDuckBot
User-Agent: ev-crawler
User-Agent: FriendlyCrawler
User-Agent: GeedoProductSearch
User-Agent: HaloBot
User-agent: iaskspider
User-Agent: IbouBot
User-agent: ICC-Crawler
User-Agent: ImagesiftBot
User-Agent: imageSpider
User-agent: img2dataset
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: LanaiBotmarch
User-agent: MatchorySearch
User-agent: MauiBot
User-Agent: MJ12bot
User-Agent: MojeekBot
User-Agent: oBot
User-agent: omgili
User-agent: omgilibot
User-Agent: OpenindexSpider
User-Agent: Orbbot
User-Agent: PanguBot
User-Agent: PerplexityBot
User-Agent: Perplexity-User
User-Agent: PetalBot
User-Agent: Qwantbot
User-Agent: SaluteBot
User-Agent: Scrapy
User-Agent: SeekportBot

# AI-powered digital marketing platform [USA]
#User-Agent: SemrushBot-BA
#User-agent: SemrushBot-OCOB
#User-agent: SemrushBot-SWA

User-agent: Sidetrade indexer bot
#User-Agent: SiteAuditBot
User-Agent: SeobilityBot
User-Agent: serpstatbot
User-Agent: SeznamBot
User-Agent: Thinkbot
User-Agent: TikTokSpider
User-Agent: Timpibot
User-Agent: Turnitin
User-Agent: TurnitinBot
User-Agent: TwitterBot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-Agent: YandexBot
User-Agent: YandexUserProxy
User-Agent: Yeti
User-Agent: YelpBot
User-Agent: YouBot
User-Agent: ZoominfoBot
Allow: /
Disallow: /amlocator/
Crawl-Delay: 100


# See also 
# https://github.com/ai-robots-txt/ai.robots.txt/blob/main/robots.txt

Sitemap: https://www.triadtechnologies.com/media/sitemap.xml