User-agent: *
Disallow: /user/
Disallow: /newposts/
Disallow: /statistics.html
Disallow: /*subaction=userinfo
Disallow: /*subaction=newposts
Disallow: /*do=lastcomments
Disallow: /*do=feedback
Disallow: /*do=register
Disallow: /*do=lostpassword
Disallow: /*do=addnews
Disallow: /*do=stats
Disallow: /*do=pm
Disallow: /*do=search
Disallow: /*do=download


User-agent: Bingbot
Allow: /
Crawl-delay: 5

User-agent: Yandex
Allow: /
Crawl-delay: 10

User-agent: Googlebot
Allow: /

Sitemap: https://softddl.org/sitemap.xml


User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeekportBot
Disallow: /

# ALL BOTS
User-agent: Pinterest
Crawl-delay: 1

# Crawlers Setup
# Block SEOkicks
User-agent: SEOkicks-Robot
Disallow: /

# Block SISTRIX
User-agent: SISTRIX Crawler
Disallow: /

# Block UptimeRobot
User-agent: UptimeRobot/2.0
Disallow: /

User-agent: 008
Disallow: /

# Block Ezooms Robot
User-agent: Ezooms Robot
Disallow: /

# Block Perl LWP
User-agent: Perl LWP
Disallow: /

# Block netEstate NE Crawler 
User-agent: netEstate NE Crawler
Disallow: /

# Block WiseGuys Robot
User-agent: WiseGuys Robot
Disallow: /