# robots.txt file for www.natur-leaf.com # Disallow indexing of specific directories User-agent: * Disallow: /general_review/ Disallow: /ingredients/ Disallow: /introduction/ Disallow: /LogFiles/ Disallow: /research/ Disallow: /townsend_letter/ Disallow: /townsend_original/ Disallow: /usage/ Disallow: /_private/ Disallow: /graphics/ # Disallow indexing of specific file extensions Disallow: /*.gif$ Disallow: /*.jpg$ Disallow: /*.zip$ # Disallow Collectors and Spam User-agent: atSpider Disallow: / User-agent: baiduspider Disallow: / User-agent: bright.net caching robot Disallow: / User-agent: Bloodhound Disallow: / User-agent: CazoodleBot Disallow: / User-agent: cherrypicker Disallow: / User-agent: Collage Disallow: / User-agent: DownLoad Express Disallow: / User-agent: Download Wonder Disallow: / User-agent: DSurf Disallow: / User-agent: EliteSys Entry Disallow: / User-agent: EmailCollector Disallow: / User-agent: e-collector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: ESIRover Disallow: / User-agent: Exabot-Images Disallow: / User-agent: Exabot Disallow: / User-agent: favorstarbot Disallow: / User-agent: flatlandbot Disallow: / User-agent: GetBot Disallow: / User-agent: GetUrl Disallow: / User-agent: Girafabot Disallow: / User-agent: HavIndex Disallow: / User-agent: httrack Disallow: / User-agent: ichiro Disallow: / User-agent: ilial Disallow: / User-agent: InfoSpiders Disallow: / User-agent: Internet Explore Disallow: / User-agent: IRLbot Disallow: / User-agent: KDD Exploror Disallow: / User-agent: lanshanbot Disallow: / User-agent: larbin Disallow: / User-agent: LightningDownload Disallow: / User-agent: LinkWalker Disallow: / User-agent: Mail Sweeper Disallow: / User-agent: Moozilla Disallow: / User-agent: MQBOT Disallow: / User-agent: munky Disallow: / User-agent: NimbleCrawler Disallow: / User-agent: NPBot Disallow: / User-agent: OCP HRS Disallow: / User-agent: panscient.com Disallow: / User-agent: PBWF Disallow: / User-agent: Picsearch Disallow: / User-agent: POE-Component-Client-HTTP Disallow: / User-agent: robofox Disallow: / User-agent: Roverbot Disallow: / User-agent: SBIder Disallow: / User-agent: Shai'Hulud Disallow: / User-agent: Snapbot Disallow: / User-agent: snap.com Disallow: / User-agent: snooper Disallow: / User-agent: SpiderBot Disallow: / User-agent: ssearcher100 Disallow: / User-agent: Templeton Disallow: / User-agent: TurnitinBot Disallow: / User-agent: vobsub Disallow: / User-agent: w3mir Disallow: / User-agent: WallPaper Disallow: / User-agent: WebBandit Disallow: / User-agent: WebCapture Disallow: / User-agent: WebCatcher Disallow: / User-agent: webcollage Disallow: / User-agent: WebCopy Disallow: / User-agent: webcopier Disallow: / User-agent: WebEmailExtrac Disallow: / User-agent: webfetcher Disallow: / User-agent: WebFetcher Disallow: / User-agent: WebMirror Disallow: / User-agent: WebReaper Disallow: / User-agent: WebWalk Disallow: / User-agent: webvac Disallow: / User-agent: wGet Disallow: / User-agent: xGet Disallow: / # Disallow Offline Browsers User-agent: Mozilla/2.0 (compatible; MS FrontPage 4.0) Disallow: /