User-Agent: Pinterestbot
Crawl-Delay: 120

User-Agent: SemrushBot
Crawl-Delay: 120

User-Agent: AhrefsBot
Crawl-Delay: 120

User-Agent: MJ12bot
Crawl-Delay: 120

User-Agent: ImagesiftBot
Crawl-Delay: 120

User-agent: SiteAuditBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SplitSignalBot
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /

User-agent: SemrushBot-FT
Disallow: /

User-agent: *
Allow: /
Disallow: /directory/

User-agent: Scrapy
User-agent: Scrapy/2.11.2
Allow: /
Disallow: /directory/