User-Agent: *
Disallow: /*.csv$
Disallow: /forgot/
Disallow: /cheesefd/
Disallow: /shopping/

# Megalodon-block
User-agent: Megalodon
Disallow: /
# archivebot-block
User-agent: archive.org_bot
User-agent: ia_archiver
Disallow: /
# Common Crawl
User-agent: CCBot
Disallow: /
# NICT
User-agent: ICC-Crawler
Disallow: /
# COMSYS
User-agent: Researchscan
Disallow: /
# IBM
User-agent: oBot
Disallow: /
# Netcraft
User-agent: NetcraftSurveyAgent
Disallow: /
# Grapeshot
User-agent: grapeshot
Disallow:
# BuiltWith
User-agent: BuiltWith
Disallow: /
# Huawei
User-agent: PetalBot
Disallow: /
User-agent: PetalBot
Allow: /w/api/
Disallow: /trap/
# AhrefsBot
User-agent: AhrefsBot
Disallow: /
# SEMrush
User-agent: SemrushBot
Disallow: /
# Steeler
User-agent: Steeler
Disallow: /
# Dotbot
User-agent: dotbot
Disallow: /
# Majestic
User-agent: MJ12bot
Disallow: /
# Serpstat
User-agent: serpstatbot
Disallow: /
# SEOkicks
User-agent: SEOkicks
Disallow: /
# Barkrowler
User-agent: barkrowler
Disallow: /
# BLEXBot
User-agent: BLEXBot
Disallow: /
# MegaIndex
User-agent: MegaIndex
Disallow: /
# proximic
User-agent: proximic
Disallow: /
# Pinterestbot
user-agent: Pinterestbot
disallow: /
# YandexBot
user-agent: YandexBot
disallow: /
# rogerbot
user-agent: rogerbot
disallow: /