User-agent: * Disallow: /visit.php Disallow: /info.php Disallow: /jewelry/buy.php Disallow: /jewelry/go.php Disallow: /magazines/shopping.php Disallow: /magazines/subscriber.php Disallow: /webdirectory/report2.php Disallow: /shopping/shoppingvisit.php Disallow: /shopping/buy.php Disallow: /cgi-bin/ Sitemap: http://www.allthewebsites.org/sitemap.xml.gz # NO access (All Spiders) User-agent: ShopWiki Disallow: / User-agent: linksmanager Disallow: / User-agent: linksmanager_bot Disallow: / User-agent: dotbot Disallow: / User-agent: grapeshot Disallow: / User-agent: MojeekBot Disallow: / User-agent: EasouSpider Disallow: / User-agent: BLEXBot Disallow: / User-agent: BacklinkCrawler Disallow: / User-agent: IntegromeDB Disallow: / User-agent: CMS Disallow: / User-agent: LinkpadBot Disallow: / User-agent: ComodoSpider Disallow: / User-agent: sistrix Disallow: / User-agent: spbot Disallow: / User-agent: CATExplorador Disallow: / User-agent: imrbot Disallow: / User-agent: Mail.RU_Bot Disallow: / User-Agent: panscient.com Disallow: / User-agent: SolomonoBot Disallow: / User-agent: NetcraftSurveyAgent Disallow: / User-agent: meanpathbot Disallow: / User-agent: 008 Disallow: / User-agent: plukkie Disallow: / User-agent: DCPbot Disallow: / User-agent: archive.org_bot Disallow: / User-agent: NCBot Disallow: / User-agent: KomodiaBot Disallow: / User-agent: aiHitBot Disallow: / User-agent: Googlebot-Image Disallow: / User-agent: SearchmetricsBot Disallow: / User-agent: CatchBot Disallow: / User-agent: Linguee Disallow: / User-agent: FlightDeckReportsBot Disallow: / User-agent: ScreenerBot Disallow: / User-agent: bingbot Disallow: / User-agent: SiteExplorer Disallow: / User-agent: Aboundexbot Disallow: / User-agent: Snipebot Disallow: / User-agent: ip-web-crawler.com Disallow: / User-agent: PagesInventory Disallow: / User-agent: 4SeoHuntBot Disallow: / User-agent: Wotbox Disallow: / User-agent: MSNBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: WBSearchBot Disallow: / User-agent: Ezooms Disallow: / User-agent: daumoa Disallow: / User-agent: Exabot Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: squirrobot Disallow: / User-agent: VoilaBot Disallow: / User-agent: NerdByNature.Bot Disallow: / User-agent: NextGenSearchBot Disallow: / User-agent: psbot Disallow: / User-agent: EventGuruBot Disallow: / User-agent: Discobot Disallow: / User-agent: GSLFbot Disallow: / User-agent: PiplBot Disallow: / User-agent: PiplBot Disallow: / User-agent: HuaweiSymantecSpider Disallow: / User-agent: Baidu Spider Disallow: / User-agent: baidu Disallow: / User-agent: baiduspider Disallow: / User-agent: baiduspider+ Disallow: / User-agent: Sosospider Disallow: / User-agent: solfo-linkchecker Disallow: / User-agent: Soso Disallow: / User-agent: Sogou Spider Disallow: / User-agent: Sogou Disallow: / User-agent: SurveyBot Disallow: / User-agent: Yandex Disallow: / User-agent: YandexBot Disallow: / User-agent: Yetibot Disallow: / User-agent: Yeti Disallow: / User-agent: Genieo Disallow: / User-agent: SeznamBot Disallow: / User-agent: ScoutJet Disallow: / User-agent: JikeSpider Disallow: / User-agent: AhrefsBot Disallow: / User-agent: Slurp Disallow: / User-agent: TurnitinBot Disallow: / User-agent: careerbot Disallow: / User-agent: wotbox Disallow: / User-agent: ProCogBot Disallow: / User-agent: Feedfetcher-Google Disallow: / User-agent: 4seohuntBot Disallow: / User-agent: CareerBot Disallow: / User-agent: Cliqusbot Disallow: / User-agent: CompSpyBot Disallow: / User-agent: DoCoMo Disallow: / User-agent: 360Spider Disallow: / User-agent: Presto Disallow: / User-agent: Jooblebot Disallow: / User-agent: TosCrawler Disallow: / User-agent: ltbot Disallow: / User-agent: DomainCrawler Disallow: / User-agent: discoverybot Disallow: / User-agent: awcheck Disallow: / User-agent: AcoonBot Disallow: / User-agent: Xenu Link Sleuth Disallow: / User-agent: Screaming Frog SEO Spider Disallow: /