# the below wildcard is intended to block bots crawling URLs that include the terms 'and' and various properties
User-agent: *
Disallow: /*-and-*-range
Disallow: /*-and-*-region
Disallow: /*-and-*-wines
Disallow: /*-and-*-spirits
Disallow: /*-and-*-beer
Disallow: /search
Disallow: /api/order
Disallow: /api/payment
Disallow: /user
Disallow: /api/*

# These user-agents are fine
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Screaming Frog SEO spider
Allow : /

User-agent: Sitebulb
Allow: /

User-agent: Mozilla/5.0 (compatible; botify; http://botify.com)
Allow: /

User-agent: Mozilla/5.0 Mobile (compatible; botify; http://botify.com)
Allow: /

# These user agents could cause unnecessary server requests
User-agent: YakazBot
Disallow: /

User-Agent: trovitBot
Disallow: /

User-Agent: Eyeotabot
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: ShopWiki
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: sistrix
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: ADmantX
Disallow: /

User-agent: YandexRenderResourcesBot/1.0
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SnapchatAds/1.0
Disallow: /

User-agent: Snap URL Preview Service; bot; snapchat
Disallow: /

User-agent: UptimeRobot/2.0
Disallow: /

User-agent: Taboolabot/3.7
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: Qwantify-prod/1.0
Disallow: /

User-agent: Brightbot 1.0
Disallow: /

User-agent: Pernod Ricard - ClickToBuy CrawlerBot/1.0
Disallow: /