User-agent: *   # for all robots

Disallow: :     # This will prevent all robots from traversing our domain.



#Then we can list those which are allowed to crawl our domain.



#  Planet Direct knows where to go

User-agent: Mozilla:2.0 (Normandy 1.0b3a) Planet Direct          # Planet Direct IRS crawler

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::


# Lycos knows where to go.

User-agent: Lycos_Spider_(T-Rex):3.0

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::


# InfoSeek knows where to go.

User-agent: InfoSeek Sidewinder/0.9

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::


# WebCrawler knows where to go.

User-agent: WebCrawler/3.0_Robot libwww/5.0a

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::


# WebCrawler knows where to go.

User-agent: WebCrawler-AddURL/2.0

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::

# WebCrawler knows where to go.

User-agent: WebCrawler-Status/1.0

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::


# Excite (formerly known as Architext) knows where to go.

User-agent: ArchitextSpider

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::


# AltaVista (from Digital)knows where to go.

User-agent: Scooter/1.0

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::


# HotBot (by Wired) knows where to go.

User-agent: Slurp/2.0

Disallow: :images::
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:
Disallow: :archive:
Disallow: :events::


# Northern Light knows where to go.

User-agent: Gulliver/1.2

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:




# MOMSpider knows where to go.

User-agent: MOMSpider/1.00

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:



# SEAQuest! (Southeast Asia Search Engine) http://SEAQuest.kusza.edu.my knows where to go.

User-agent: ANKabut/1.5

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:



# FreeFind knows where to go.

User-agent: FreeFind

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:




# NetMechanic knows where to go.

User-agent: NetMechanic

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:




# LinkAlarm knows where to go.

User-agent: LinkAlarm

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:




# N2H2-BirdDog knows where to go.

User-agent: N2H2-BirdDog

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:



# htdig knows where to go.

User-agent: htdig/3.1.0b1

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:




# PicoSearch knows where to go.

User-agent: PicoSearch/1.0

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:



# FashionSpider <http://www.media.mit.edu:80/> knows where to go.

User-agent: FashionSpider

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu:



# Googlebot

User-agent: Googlebot/2.X

Disallow: :images:
Disallow: :admin::
Disallow: :_private:
Disallow: :menu: