Sitemap: https://www.yachtcharterfleet.com/sitemap.xml
User-agent: *
Allow: /
Disallow: /my-account/
Disallow: /feeds/
Disallow: /yachts/save_search.htm
Disallow: /events/map.htm
Disallow: /directory/compare-yachts.htm
Disallow: /yachts/pdf-brochure/
Disallow: /yachts/save_search.htm
Disallow: /yachts/yacht-shortlist.htm
Disallow: /instagram/embed_proxy

#block 80legs / crawler4j spiders
User-agent: 008
User-agent: crawler4j
User-agent: crawler4j (http://code.google.com/p/crawler4j/)
User-agent: Curious George - www.analyticsseo.com/crawler
User-agent: Curious George - www.analyticsseo.com
User-agent: MegaIndex.ru/2.0
User-agent: MegaIndex.ru/
User-agent: +https://www.megaindex.ru/
User-agent: Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)
User-agent: MJ12
User-agent: Mozilla/5.0 (compatible; Steeler/3.5; http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)
User-agent: Steeler
User-agent: Microsoft-WebDAV-MiniRedir/6.1.7601
User-agent: Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)
User-agent: Findxbot
User-agent: Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)
User-agent: SeznamBot
User-agent: Seznam
User-agent: Mozilla/4.0 (compatible;  Vagabondo/4.0; http://webagent.wise-guys.nl/)
User-agent: Mozilla/4.0 (compatible;  Vagabondo/4.0; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)
User-agent: Vagabondo
User-agent: Jakarta Commons-HttpClient/3.0.1
User-agent: LWP::Simple/5.822
User-agent: Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
User-agent: SMTBot
User-agent: Mozilla/5.0 (compatible; memoryBot/1.22.56 +http://internetmemory.org/en/)
User-agent: memoryBot
User-agent: Mozilla/5.0 (compatible; smrjbot/0.0.20)
User-agent: smrjbot
User-agent: Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )
User-agent: spbot
User-agent: MauiBot (crawler.feedback+dc@gmail.com)
User-agent: ia_archiver
User-agent: BLEXBot
Disallow:/