User-agent: 360Spider
User-agent: A6-Indexer
User-agent: Adsbot
User-agent: AhrefsBot
User-agent: aiHitBot
User-agent: AlphaSeoBot
User-agent: AlphaSeoBot-SA
User-agent: AwarioSmartBot
User-agent: BDCbot
User-agent: Bleriot
User-agent: BLEXBot
User-agent: Baiduspider
User-agent: Barkrowler
User-agent: BingPreview
User-agent: Buck
User-agent: Bytespider
User-agent: CCBot
User-agent: Clickagy Intelligence Bot
User-agent: Clickagy Intelligence Bot v2
User-agent: Cliqzbot
User-agent: coccocbot-web
User-agent: DataForSeoBot
User-agent: DAUM
User-agent: DomainCrawler
User-agent: DotBot
User-agent: ExtLinksBot
User-agent: Gigabot
User-agent: GrapeshotCrawler
User-agent: HybridBot
User-agent: IndeedBot
User-agent: IstellaBot
User-agent: Keybot
User-agent: Linespider
User-agent: LinkpadBot
User-agent: LumtelBot
User-agent: ltx71
User-agent: magpie-crawler
User-agent: Mail.RU_Bot
User-agent: MauiBot
User-agent: MaxPointCrawler
User-agent: MJ12bot
User-agent: MojeekBot
User-agent: MTRobot
User-agent: Naver
User-agent: NaverBot
User-agent: Neevabot
User-agent: NetSeer
User-agent: Nikto
User-agent: Nutch
User-agent: PetalBot
User-agent: Plukkie
User-agent: proximic
User-agent: Qwantify
User-agent: Qwant-news
User-agent: RyteBot
User-agent: rogerbot
User-agent: Scrapy
User-agent: Seekport Crawler
User-agent: Semrush
User-agent: SemrushBot
User-agent: SEOkicks
User-agent: SiteExplorer
User-agent: Slurp
User-agent: SMTBot
User-agent: spbot
User-agent: Spiderbot
User-agent: Spiderbot/Nutch-1.7
User-agent: The Knowledge AI
User-agent: ToutiaoSpider
User-agent: Yahoo Ad Monitoring
User-agent: Yandex
User-agent: dataprovider
Disallow: /

User-agent: AdsBot-Google
User-agent: AppleBot
User-agent: Bingbot
User-agent: Google
User-agent: Googlebot
User-agent: Google-bot
User-agent: Googlebot-mobile
User-agent: Mozilla/5.0 (compatible;contxbot/1.0)
User-agent: Mediapartners
User-agent: Mediapartners-Google
User-agent: MSNBot
User-agent: MSNBot-Media
User-agent: Pinterestbot
User-agent: SeznamBot
User-agent: Twitterbot
Disallow: /_cron/*
Disallow: /admin/*
Disallow: /ads/*
Disallow: /random/*
Disallow: /feed/*
Disallow: /files/*
Disallow: /find/*
Disallow: /related/*
Disallow: /contact
Disallow: /logout
Disallow: /user
Disallow: /404
Crawl-delay: 5
Request-rate: 1/5s

User-agent: *
Disallow: /

# sitemaps
Sitemap: https://www.effectsdatabase.com/sitemap.xml
Sitemap: https://www.effectsdatabase.com/sitemap_categories.xml
Sitemap: https://www.effectsdatabase.com/sitemap_categories2.xml