robot-id: slurp
robot-name: Inktomi Slurp
robot-cover-url: http://www.inktomi.com/
robot-details-url: http://www.inktomi.com/slurp.html
robot-owner-name: Inktomi Corporation
robot-owner-url: http://www.inktomi.com/
robot-owner-email: slurp@inktomi.com
robot-status: active
robot-purpose: indexing, statistics
robot-type: standalone
robot-platform: unix
robot-availability: none
robot-exclusion: yes
robot-exclusion-useragent: slurp
robot-noindex: yes
robot-host: *.inktomi.com
robot-from: yes
robot-useragent: Slurp/2.0
robot-language: C/C++
robot-description: Indexing documents for the HotBot search engine
		(www.hotbot.com), collecting Web statistics
robot-history: Switch from Slurp/1.0 to Slurp/2.0 November 1996
robot-environment: service
modified-date: Fri Feb 28 13:57:43 PST 1997
modified-by: slurp@inktomi.com

robot-id: askjeeves
robot-name: AskJeeves
robot-cover-url: http://www.ask.com
robot-details-url: 
robot-owner-name: Ask Jeeves, Inc.
robot-owner-url: http://www.ask.com
robot-owner-email: postmaster@ask.com
robot-status: active
robot-purpose: indexing, maintenance
robot-type: standalone
robot-platform: linux
robot-availability: none
robot-exclusion: yes
robot-exclusion-useragent: "Teoma" or "Ask Jeeves" or "Jeeves"
robot-noindex: Yes
robot-host: ez*.directhit.com
robot-from: No
robot-useragent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma) 
robot-language: c++
robot-description: Ask Jeeves / Teoma spider
robot-history: Developed by Direct Hit Technologies which was aquired by
 Ask Jeeves in 2000.
robot-environment: service
modified-date: Fri Jan 17 15:20:08 EST 2003
modified-by: brucep@ask.com

obot-id: googlebot
robot-name: Googlebot
robot-cover-url: http://www.googlebot.com/ 
robot-details-url: http://www.googlebot.com/bot.html
robot-owner-name: Google Inc.
robot-owner-url: http://www.google.com/
robot-owner-email: googlebot@google.com 
robot-status: active
robot-purpose: indexing
robot-type: standalone 
robot-platform: Linux
robot-availability: none
robot-exclusion: yes
robot-exclusion-useragent: googlebot
robot-noindex: yes
robot-host: googlebot.com
robot-from: yes 
robot-useragent: Googlebot/2.X (+http://www.googlebot.com/bot.html)
robot-language: c++
robot-description: Google's crawler
robot-history: Developed by Google Inc
robot-environment: commercial
modified-date: Thu Mar 29 21:00:07 PST 2001
modified-by: googlebot@google.com

robot-id:                       bbot
robot-name:                     BBot
robot-cover-url:                http://www.otthon.net/search
robot-details-url:              http://www.otthon.net/search/bbot
robot-owner-name:               Istvan Fulop
robot-owner-url:                http://www.otthon.net
robot-owner-email:              poluf1 at yahoo dot co dot uk
robot-status:                   development
robot-purpose:                  indexing, maintenance
robot-type:                     standalone
robot-platform:                 windows
robot-availability:             none
robot-exclusion:                yes
robot-exclusion-useragent:      bbot
robot-noindex:                  yes
robot-nofollow:			yes
robot-host:                     *.netcologne.de
robot-from:                     yes
robot-useragent:                bbot/0.100
robot-language:                 perl
robot-description:              Mainly intended for site level search, sometimes set loose.
robot-history:                  Started project in 11/2000. Called BBot since 24/04/2003.
robot-environment:              hobby
modified-date:                  Sun, 04 May 2003 10:15:00 GMT
modified-by:                    Istvan Fulop

robot-id: scooter
robot-name: Scooter
robot-cover-url: http://www.altavista.com/
robot-details-url: http://www.altavista.com/av/content/addurl.htm
robot-owner-name: AltaVista
robot-owner-url: http://www.altavista.com/
robot-owner-email: scooter@pa.dec.com
robot-status: active
robot-purpose: indexing
robot-type: standalone
robot-platform: unix
robot-availability: none
robot-exclusion: yes
robot-exclusion-useragent: Scooter
robot-noindex: yes
robot-host: *.av.pa-x.dec.com
robot-from: yes
robot-useragent: Scooter/2.0 G.R.A.B. V1.1.0
robot-language: c
robot-description: Scooter is AltaVista's prime index agent.
robot-history: Version 2 of Scooter/1.0 developed by Louis Monier of WRL.
robot-environment: service
modified-date: Wed, 13 Jan 1999 17:18:59 GMT
modified-by: steves@avs.dec.com

robot-id: msnbot
robot-name: MSNBot
robot-cover-url: http://search.msn.com
robot-details-url: http://search.msn.com/msnbot.htm
robot-owner-name: Microsoft Corp.
robot-owner-url: http://www.microsoft.com
robot-owner-email: msnbot@microsoft.com
robot-status: active
robot-purpose: indexing
robot-type: standalone
robot-platform: Windows Server 2000, Windows Server 2003
robot-availability: none
robot-exclusion: yes
robot-exclusion-useragent: msnbot
robot-noindex: yes
robot-host: <TBD>
robot-from: yes
robot-useragent: MSNBOT/0.1 (http://search.msn.com/msnbot.htm)
robot-language: C++
robot-description: MSN Search Crawler
robot-history: Developed by Microsoft Corp.
robot-environment: commercial
modified-date: June 23, 2003
modified-by: msnbot@microsoft.com