User-agent: * 

Crawl-delay: 5 # задает таймаут в 5 секунды

User-agent: Slurp
Disallow: /    

User-agent: yahoo-mmcrawler 
Disallow: / 

User-agent: Yahoo! Slurp 
Disallow: / 

User-agent: YahooSeeker 
Disallow: /

User-agent: Aport
Disallow: /

User-agent: Webalta
Disallow: /