User-agent: *
Disallow: /adSys
Disallow: /mredir
Disallow: /sredir
Disallow: /clickout

User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /

User-agent: SurveyBot
Disallow: /

User-agent: Heritrix
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: AboutUsBot 
Disallow: / 

User-agent: Gigabot 
Disallow: / 

User-agent: SapphireWebCrawler 
Disallow: / 

User-agent: Yandex 
Disallow: /