#on Dev. for production
User-agent: *
Disallow: /*/system
Disallow: /*/data
Disallow: /*/search
Disallow: /*/search
Disallow: /*/products/search/
Disallow: /*/admin/
Crawl-delay:10

#media.newzealand.com - /robots.txt

Sitemap: http://media.newzealand.com/en/sitemap-xml

#Prevent all tag intersections from being indexed
#Disallow: /*+

# Access denied

User-agent: *Fetch* 
Disallow: /

User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /

User-agent: BorderManager*
Disallow: /

User-agent: webcollage*
Disallow: /

User-agent: Java*
Disallow: /

User-agent: grub-client 
Disallow: /

User-agent: lwp*
Disallow: /

User-agent: *TunnelPro*
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: larbin
Disallow: /

User-agent: 008
Disallow: /