# robots.txt

User-agent: *
Disallow: /admin
Disallow: /config
Disallow: /css
Disallow: /fry_include
Disallow: /global
Disallow: /includes
Disallow: /js
Disallow: /manager
Disallow: /notfound.htm
Disallow: /prod
Disallow: /utils
Disallow: /xml
Disallow: /xsl
Disallow: /checkout
Disallow: /user
Crawl-Delay: 1  	# 1 second delay between pages
Request-rate: 1/1  	# maximum of 1 page per second
Vsit-time: 1300-2100	# only allow scans between 13:00 and 21:00 UTC

User-agent: JetSetter
Disallow: /