# This file is now maintained in Github
# Version: v6.283.5807

#####################################################
# Bad bots
User-agent: dotbot
Disallow: /

User-agent: Fasterfox
Disallow: /

User-agent: WiseBot
Disallow: /

User-agent: ConveraCrawler
Disallow: /

User-agent: Scrubby
Disallow: /

User-agent: Robozilla
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: psbot
Disallow: /

User-agent: asterias
Disallow: /

User-agent: CsCrawler
Disallow: /

User-agent: Larbin
Disallow: /

User-agent: Sproose
Disallow: /

User-agent: NaverBot
Disallow: /

User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: wget
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: k2spider
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: reloado.com SearchEngine
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Fiddler
Disallow: /

User-agent: CloudTV
Disallow: /

User-agent: Buck
Disallow: /

User-agent: TurnitinBot
Disallow: /

# IA_archiver
User-agent: ia_archiver
Disallow: /
User-agent: ia_archiver-web.archive.org
Disallow: /

#####################################################
# Zaphod bots
User-agent: Twitterbot
User-agent: facebookexternalhit
Crawl-delay: 42
Sitemap: https://www.arte.tv/sitemap.xml
Disallow: /admin/
Disallow: /api/mami/
Disallow: /cgi-bin/
Disallow: /club-preprod/
Disallow: /digital-preprod/
Disallow: /hbbtvv2-preprod/
Disallow: /magazine-dev/
Disallow: /magazine-test/
Disallow: /player-preprod/
Disallow: /question-preprod/
Disallow: /replay-preprod/
Disallow: /resize-preprod/
Disallow: /resize/
Disallow: /search/
Disallow: /search?q=
Disallow: /sitespreprod/
Disallow: /static-preprod/
Disallow: /static/Tsunami/
Disallow: /static/c0/
Disallow: /static/c1/
Disallow: /static/c2/
Disallow: /static/c3/
Disallow: /static/c4/
Disallow: /static/c5/
Disallow: /static/c6/
Disallow: /static/c7/
Disallow: /static/plokker/
Disallow: /static/u1/
Disallow: /static/u2/
Disallow: /static/u3/
Disallow: /static/u4/
Disallow: /static/u5/
Disallow: /static/u6/


#####################################################
# Good bots
User-agent: Googlebot
User-agent: bingbot
User-agent: Yandex
User-agent: Qwantify
User-agent: DuckDuckBot
Crawl-delay: 2
Sitemap: https://www.arte.tv/sitemap.xml
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /club-preprod/
Disallow: /digital-preprod/
Disallow: /hbbtvv2-preprod/
Disallow: /magazine-dev/
Disallow: /magazine-test/
Disallow: /player-preprod/
Disallow: /question-preprod/
Disallow: /replay-preprod/
Disallow: /resize-preprod/
Disallow: /search/
Disallow: /search?q=
Disallow: /sitespreprod/
Disallow: /static-preprod/
Disallow: /static/Tsunami/
Disallow: /static/c0/
Disallow: /static/c1/
Disallow: /static/c2/
Disallow: /static/c3/
Disallow: /static/c4/
Disallow: /static/c5/
Disallow: /static/c6/
Disallow: /static/c7/
Disallow: /static/plokker/
Disallow: /static/u1/
Disallow: /static/u2/
Disallow: /static/u3/
Disallow: /static/u4/
Disallow: /static/u5/
Disallow: /static/u6/

#####################################################
# Ugly bots and manically depressed robots deserve a long crawl delay  
User-agent: *
Crawl-delay: 42
Sitemap: https://www.arte.tv/sitemap.xml
Disallow: /admin/
Disallow: /api/mami/
Disallow: /cgi-bin/
Disallow: /club-preprod/
Disallow: /digital-preprod/
Disallow: /hbbtvv2-preprod/
Disallow: /magazine-dev/
Disallow: /magazine-test/
Disallow: /player-preprod/
Disallow: /question-preprod/
Disallow: /replay-preprod/
Disallow: /resize-preprod/
Disallow: /resize/
Disallow: /search/
Disallow: /search?q=
Disallow: /sitespreprod/
Disallow: /static-preprod/
Disallow: /static/Tsunami/
Disallow: /static/c0/
Disallow: /static/c1/
Disallow: /static/c2/
Disallow: /static/c3/
Disallow: /static/c4/
Disallow: /static/c5/
Disallow: /static/c6/
Disallow: /static/c7/
Disallow: /static/plokker/
Disallow: /static/u1/
Disallow: /static/u2/
Disallow: /static/u3/
Disallow: /static/u4/
Disallow: /static/u5/
Disallow: /static/u6/