User-agent: *
Allow: /
Disallow: /www.youtube.com/

User-agent: AhrefsBot
User-agent: AhrefsBot/4.0
User-agent: BacklinkCrawler
User-agent: GSiteCrawler
User-agent: ip-web-crawler.com
User-agent: ips-agent
User-agent: Linkdex
User-agent: linkdex.com
User-agent: linkdex.com/v2.0
User-agent: linkdex.com/v2.1
User-agent: MJ12bot
User-agent: SpyFu
User-agent: Xenu
User-agent: Xenu Link Sleuth
User-agent: dotbot
User-Agent: MJ12bot/v1.4.3
User-agent: plukkie
Disallow: /

User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
User-agent: Baiduspider+
User-agent: Baiduspider+(+http://www.baidu.com/search/spider.htm)
User-agent: Baiduspider/2.0;+http://www.baidu.com/search/spider.html
User-agent: Baiduspider/2.0
User-agent: +Baiduspider
User-agent: +Baiduspider/2.0
User-agent: +Baiduspider/2.0;++http://www.baidu.com/search/spider.html
User-agent: Mozilla/5.0(compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
User-agent: EasouSpider
User-agent: JikeSpider
User-agent: Sosospider+
User-agent: Sosospider
User-agent: Sosospider/2.0
User-agent: 360Spider
User-agent: YisouSpider
User-agent: YoudaoBot/1.0
User-agent: YoudaoBot/1.0
User-agent: sogou spider
Disallow: /

User-agent: Mediapartners-Google
User-agent: BLEXBot
User-agent: VelenPublicWebCrawler
User-agent: SemrushBot
Disallow: /