# robots.txt for http://aghsreunions.com

# We have attempted to make it easier for spiders to quickly bypass
# invalid and useless pages. Even though no harm likely will come to
# our site by spiders reading every directory, they can save a lot
# of time and bandwidth (and save it for us, too) by respecting this
# robots.txt file and the directories and files we have excluded.
# Thanks!

# Please note: There are a lot of pages on this site, and there are
# some misbehaved spiders out there that go _way_ too fast.
# If you're irresponsible, your access to the site may be blocked.
# Persistent unauthorized bots may be banned by host and/or IP.

# Inktomi's "Slurp" can read a minimum delay between hits; if your
# bot supports such a thing using the 'Crawl-delay' or another
# instruction, please let us know.
## *At least* a 2 second please, preferably more :)
Crawl-delay: 2

# Pompos (and others) use the 'revisit-after' command in either
# meta tag format or in robots.txt. This causes the bot to wait
# at least a certain amount of time before crawling again.
# Months:  revisit-after 2 months: /path*
# Days:    revisit-after 7: /path*
# Minutes: revisit-after 100 mins: /
revisit-after 1: /wiki*
revisit-after 1: /blog*
revisit-after 3: /plaza*
revisit-after 7: /gallery*
revisit-after 3: /

# Advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /
User-agent: CJNetworkQuality
Disallow: /

# Image-harvesting bots:
User-agent: Googlebot-Image
Disallow: /
User-agent: IconSurf
Disallow: /

# Wikipedia work bots:
User-agent: IsraBot
Disallow:
User-agent: Orthogaffe
Disallow:

# Crawlers that are kind enough to obey, but which we'd
# rather not have unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Zao
Disallow: /

# Unconfigured bots.
User-agent: Your Name Here
Disallow: /
User-agent: ISSpider-3.0
Disallow: /

# Some bots are known to be trouble, particularly those
# designed to copy entire sites. Please respect robots.txt.
User-agent: Download Ninja
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: linko
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: ZyBORG
Disallow: /

# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
# for instance.
User-agent: wget
Disallow: /

# The 'grub' distributed client has been *very* poorly behaved.
User-agent: grub-client
Disallow: /

# Doesn't follow robots.txt anyway, but...
User-agent: k2spider
Disallow: /

# Hits many times per second, not acceptable
# (http://www.nameprotect.com/botinfo.html)
User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with
# no public benefit (http://www.webreaper.net/)
User-agent: WebReaper
Disallow: /

# Don't allow the wayback-machine to index user-pages
#User-agent: ia_archiver
#Disallow: /wiki/User

# Friendly, low-speed bots are welcome to view most site pages,
# but not dynamically-generated pages please.
User-agent: *					# Apply to all robots
Disallow: /go					# Dynamic redirects
Disallow: /w/					# Dynamically-generated Wiki scripts (not the content)

# Static non-content folders and files (mostly older stuff):
Disallow: /cache/				# Cache files
Disallow: /css/					# Stylesheets - Not currently used
Disallow: /errors/				# Error pages
Disallow: /files/				# File library - Not currently used
Disallow: /icons/				# Shared icons - Not currently used
Disallow: /images/				# Images
Disallow: /links/				# List of links
Disallow: /logs/				# Log files
Disallow: /mail/				# Mail files
Disallow: /media/				# Media files
Disallow: /scripts/				# JavaScript and PHP scripts
Disallow: /stats/				# Statistics
Disallow: /trap/				# Badly-behaved bot trap

# Block blog directories and files that shouldn't be spidered
Disallow: /blog/wp-admin/
Disallow: /blog/wp-content/
Disallow: /blog/wp-includes/
Disallow: /blog.old/
Disallow: /blog.old.1/
Disallow: /blog.old.2/
Disallow: /blog.old.3/
Disallow: /blog.old.4/
Disallow: /blog.old.5/
Disallow: /blog/info.txt
Disallow: /blog/license.txt
Disallow: /blog/readme.html

# Block forum directories and files that shouldn't be spidered
Disallow: /community/admin/
Disallow: /community/cache/
Disallow: /community/db/
Disallow: /community/docs/
Disallow: /community/images/
Disallow: /community/includes/
Disallow: /community/language/
Disallow: /community/templates/
Disallow: /community.old/
Disallow: /community.old.1/
Disallow: /community.old.2/
Disallow: /community.old.3/
Disallow: /community.old.4/
Disallow: /community.old.5/

# Block gallery directories and files that shouldn't be spidered
Disallow: /gallery/images/
Disallow: /gallery/install/
Disallow: /gallery/lib/
Disallow: /gallery/modules/
Disallow: /gallery/themes/
Disallow: /gallery/upgrade/
Disallow: /gallery.old/
Disallow: /gallery.old.1/
Disallow: /gallery.old.2/
Disallow: /gallery.old.3/
Disallow: /gallery.old.4/
Disallow: /gallery.old.5/
Disallow: /gallery/LICENSE
Disallow: /gallery/MANIFEST
Disallow: /gallery/README.html
Disallow: /gallery/login.txt

# Block plaza directories and files that shouldn't be spidered
Disallow: /plaza/admin/
Disallow: /plaza/auth/
Disallow: /plaza/backup/
Disallow: /plaza/blocks/
Disallow: /plaza/blog/
Disallow: /plaza/calendar/
Disallow: /plaza/course/
Disallow: /plaza/enrol/
Disallow: /plaza/error/
Disallow: /plaza/files/
Disallow: /plaza/filter/
Disallow: /plaza/grade/
Disallow: /plaza/group/
Disallow: /plaza/install/
Disallow: /plaza/iplookup/
Disallow: /plaza/lang/
Disallow: /plaza/lib/
Disallow: /plaza/login/
Disallow: /plaza/message/
Disallow: /plaza/mnet/
Disallow: /plaza/mod/
Disallow: /plaza/my/
Disallow: /plaza/pix/
Disallow: /plaza/question/
Disallow: /plaza/search/
Disallow: /plaza/sso/
Disallow: /plaza/theme/
Disallow: /plaza/user/
Disallow: /plaza/userpix/
Disallow: /plaza.old/
Disallow: /plaza.old.1/
Disallow: /plaza.old.2/
Disallow: /plaza.old.3/
Disallow: /plaza.old.4/
Disallow: /plaza.old.5/
Disallow: /plaza/README.txt
Disallow: /plaza/config-dist.php
Disallow: /plaza/config.php
Disallow: /plaza/install.php
Disallow: /plaza/tags
Disallow: /plaza/version.php

# Block wiki directories and files that shouldn't be spidered
Disallow: /wiki/bin/
Disallow: /wiki/config/
Disallow: /wiki/docs/
Disallow: /wiki/extensions/
Disallow: /wiki/images/
Disallow: /wiki/includes/
Disallow: /wiki/languages/
Disallow: /wiki/locale/
Disallow: /wiki/maintenance/
Disallow: /wiki/math/
Disallow: /wiki/skins/
Disallow: /wiki/tests/
Disallow: /wiki/uploads/
Disallow: /wiki.old/
Disallow: /wiki.old.1/
Disallow: /wiki.old.2/
Disallow: /wiki.old.3/
Disallow: /wiki.old.4/
Disallow: /wiki.old.5/
Disallow: /wiki/AdminSettings.php
Disallow: /wiki/AdminSettings.sample
Disallow: /wiki/COPYING
Disallow: /wiki/FAQ
Disallow: /wiki/HISTORY
Disallow: /wiki/INSTALL
Disallow: /wiki/LocalSettings.php
Disallow: /wiki/Makefile
Disallow: /wiki/README
Disallow: /wiki/RELEASE-NOTES
Disallow: /wiki/StartProfiler.php
Disallow: /wiki/Test.php
Disallow: /wiki/info.txt

# Block wiki namespaces we don't want spidered
Disallow: /wiki/Media
Disallow: /wiki/Image
Disallow: /wiki/Template

# Block wiki deletion and other debates discussion
Disallow: /wiki/AGHS_Reunions_Wiki:Articles_for_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki%3AArticles_for_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki:Copyright_problems
Disallow: /wiki/AGHS_Reunions_Wiki%3ACopyright_problems
Disallow: /wiki/AGHS_Reunions_Wiki:Miscellaneous_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki%3AMiscellaneous_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki:Miscellany_for_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki%3AMiscellany_for_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki:Pages_for_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki%3APages_for_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki:Protected_titles/
Disallow: /wiki/AGHS_Reunions_Wiki%3AProtected_titles/
Disallow: /wiki/AGHS_Reunions_Wiki:Requests_for_arbitration/
Disallow: /wiki/AGHS_Reunions_Wiki%3ARequests_for_arbitration/
Disallow: /wiki/AGHS_Reunions_Wiki:Requests_for_comment/
Disallow: /wiki/AGHS_Reunions_Wiki%3ARequests_for_comment/
Disallow: /wiki/AGHS_Reunions_Wiki:Requests_for_adminship/
Disallow: /wiki/AGHS_Reunions_Wiki%3ARequests_for_adminship/
Disallow: /wiki/AGHS_Reunions_Wiki:Votes_for_deletion/
Disallow: /wiki/AGHS_Reunions_Wiki%3AVotes_for_deletion/

# Block public talk wiki namespaces to avoid indexing spam comments
Disallow: /wiki/Talk
Disallow: /wiki/User_talk
Disallow: /wiki/User%20talk
Disallow: /wiki/AGHS_Reunions_Wiki_talk
Disallow: /wiki/AGHS%20Reunions%20Wiki%20talk
Disallow: /wiki/Image_talk
Disallow: /wiki/Image%20talk
Disallow: /wiki/MediaWiki_talk
Disallow: /wiki/MediaWiki%20talk
Disallow: /wiki/Template_talk
Disallow: /wiki/Template%20talk
Disallow: /wiki/Help_talk
Disallow: /wiki/Help%20talk
Disallow: /wiki/Category_talk
Disallow: /wiki/Category%20talk
Disallow: /wiki/Portal_talk
Disallow: /wiki/Portal%20talk
Disallow: /wiki/WikiProject_talk
Disallow: /wiki/WikiProject%20talk
Disallow: /wiki/Forms_talk
Disallow: /wiki/Forms%20talk

# Block other administrative wiki pages (shouldn't have access anyway)
Disallow: /wiki/Special:Blockip
Disallow: /wiki/Special%3ABlockip
Disallow: /wiki/Special:DeletedContributions
Disallow: /wiki/Special%3ADeletedContributions
Disallow: /wiki/Special:Makebot
Disallow: /wiki/Special%3AMakebot
Disallow: /wiki/Special:Import
Disallow: /wiki/Special%3AImport
Disallow: /wiki/Special:Makesysop
Disallow: /wiki/Special%3AMakesysop
Disallow: /wiki/Special:Renameuser
Disallow: /wiki/Special%3ARenameuser
Disallow: /wiki/Special:Unwatchedpages
Disallow: /wiki/Special%3AUnwatchedpages
Disallow: /wiki/Special:Undelete
Disallow: /wiki/Special%3AUndelete

# Block search and random wiki pages
Disallow: /wiki/Special:Linksearch
Disallow: /wiki/Special%3ALinksearch
Disallow: /wiki/Special:Random
Disallow: /wiki/Special%3ARandom
Disallow: /wiki/Special:Randomredirect
Disallow: /wiki/Special%3ARandomredirect
Disallow: /wiki/Special:Search
Disallow: /wiki/Special%3ASearch

# Block other Special wiki pages that are not useful to search engines
Disallow: /wiki/Special:Allmessages
Disallow: /wiki/Special%3AAllmessages
Disallow: /wiki/Special:Booksources
Disallow: /wiki/Special%3ABooksources
Disallow: /wiki/Special:BrokenRedirects
Disallow: /wiki/Special%3ABrokenRedirects
Disallow: /wiki/Special:CategoryTree
Disallow: /wiki/Special%3ACategoryTree
Disallow: /wiki/Special:Cite
Disallow: /wiki/Special%3ACite
Disallow: /wiki/Special:Contributions
Disallow: /wiki/Special%3AContributions
Disallow: /wiki/Special:DoubleRedirects
Disallow: /wiki/Special%3ADoubleRedirects
Disallow: /wiki/Special:DynamicPageListSP
Disallow: /wiki/Special%3ADynamicPageListSP
Disallow: /wiki/Special:ExpandTemplates
Disallow: /wiki/Special%3AExpandTemplates
Disallow: /wiki/Special:Export
Disallow: /wiki/Special%3AExport
Disallow: /wiki/Special:Ipblocklist
Disallow: /wiki/Special%3AIpblocklist
Disallow: /wiki/Special:Userlogin
Disallow: /wiki/Special%3AUserlogin
Disallow: /wiki/Special:Log
Disallow: /wiki/Special%3ALog
Disallow: /wiki/Special:MIMEsearch
Disallow: /wiki/Special%3AMIMEsearch
Disallow: /wiki/Special:Preferences
Disallow: /wiki/Special%3APreferences
Disallow: /wiki/Special:Prefixindex
Disallow: /wiki/Special%3APrefixindex
Disallow: /wiki/Special:Unusedimages
Disallow: /wiki/Special%3AUnusedimages
Disallow: /wiki/Special:Unusedtemplates
Disallow: /wiki/Special%3AUnusedtemplates
Disallow: /wiki/Special:Upload
Disallow: /wiki/Special%3AUpload
Disallow: /wiki/Special:Version
Disallow: /wiki/Special%3AVersion
Disallow: /wiki/Special:Wantedcategories
Disallow: /wiki/Special%3AWantedcategories
Disallow: /wiki/Special:Wantedpages
Disallow: /wiki/Special%3AWantedpages
Disallow: /wiki/Special:Watchlist
Disallow: /wiki/Special%3AWatchlist
Disallow: /wiki/Special:Withoutinterwiki
Disallow: /wiki/Special%3AWithoutinterwiki