# # blank file allows all
# 
# # Let's add in some stuff
# 
# User-agent: sembot
# Disallow: /
# 
# User-agent: *
# Disallow: /wp-admin/
# 
# User-agent: DataForSeoBot
# Disallow: /
# 
# User-agent: Bytedance
# Disallow: /

#
#  robots.txt file for www.firstgreatwestern.info
#
#  we encourage robots to visit and index all documents
#  but not any executable scripts. 
#
#  from an original of January 2005
#  Most recent update - Nov 2011 - Adde Baiduspider
#  Most recent update - Aug 2020 - Added Semrushbot 
#
# This site is provided by a customer of First Great Western, and the views expressed 
# are those of the individual contributors concerned.

# 6.2.2020 adde "dated" to stop indexing of older stuff
# /smr added 5.2024 

User-agent:  *
Disallow: /cgi-bin/
Disallow: /sec
Disallow: /dated
Disallow: /mirror
Disallow: /smr

User-agent: Baiduspider 
Disallow: /

User-agent: AhrefsBot 
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Crawl-delay: 15

User-Agent: MJ12bot
Crawl-Delay: 5

User-Agent: trendictionbot
Crawl-Delay: 5