# These rules apply to all spiders.
User-agent: *
# These limit the actions that the spiders will preform (i.e., because
# /twiki/bin/rdiff is disallowed, they won't try to visit old revisions)
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/rdiffauth
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/savemulti
Disallow: /twiki/bin/search
Disallow: /twiki/bin/setlib.cfg
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/testenv
Disallow: /twiki/bin/configure
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/bin/viewfile
Disallow: /twiki/list/
# These limit the sections of the site that the robots will visit.
# There really is no reason to have the robots visit these webs; they
# don't really have any content.
Disallow: /twiki/bin/view/Trash/
Disallow: /twiki/bin/view/TWiki/
Disallow: /twiki/bin/view/Sandbox/

# These rules only apply to Google's spiders. Google has some extra
# features, like pattern matching.
User-agent: Googlebot
Disallow: /twiki/*WebAtom*
Disallow: /twiki/*WebChanges*
Disallow: /twiki/*WebIndex*
Disallow: /twiki/*WebLeftBar*
Disallow: /twiki/*WebNotify*
Disallow: /twiki/*WebPreferences*
Disallow: /twiki/*WebRss*
Disallow: /twiki/*WebSearch*
Disallow: /twiki/*WebSearchAdvanced*
Disallow: /twiki/*WebStatistics*
Disallow: /twiki/*WebTopicCreator*
Disallow: /twiki/*WebTopicList*
Disallow: /twiki/*?cover=print*
Disallow: /twiki/*?rev=*
Disallow: /twiki/*?raw=on*
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/rdiffauth
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/savemulti
Disallow: /twiki/bin/search
Disallow: /twiki/bin/setlib.cfg
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/testenv
Disallow: /twiki/bin/configure
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/list/
# These limit the sections of the site that the robots will visit.
# There really is no reason to have the robots visit these webs; they
# don't really have any content.
Disallow: /twiki/bin/view/Trash/
Disallow: /twiki/bin/view/TWiki/
Disallow: /twiki/bin/view/Sandbox/

User-agent: Slurp
Crawl-delay: 10
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/rdiffauth
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/savemulti
Disallow: /twiki/bin/search
Disallow: /twiki/bin/setlib.cfg
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/testenv
Disallow: /twiki/bin/configure
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/bin/viewfile
Disallow: /twiki/list/
# These limit the sections of the site that the robots will visit.
# There really is no reason to have the robots visit these webs; they
# don't really have any content.
Disallow: /twiki/bin/view/Trash/
Disallow: /twiki/bin/view/TWiki/
Disallow: /twiki/bin/view/Sandbox/