User-Agent: *# Disallow all crawlers access to certain pages.
Disallow: /images/
Disallow: /live/

Allow: /

User-agent: Mediapartners-Google
Allow: /
 
User-agent: Adsbot-Google
Allow: /
 
User-agent: Googlebot-Image
Allow: /
 
User-agent: Googlebot-Mobile
Allow: /
 
User-agent: ia_archiver-web.archive.org
Allow: /