# $CVSHeader: ralf/rowanthorpe.com/robots.txt,v 1.2 2009-09-16 00:07:30 rowan Exp $
User-agent: WDG_SiteValidator

Disallow: /files/
Disallow: /images/
Disallow: /logs/
Disallow: /media/
Disallow: /offline/
Disallow: /progs/

User-agent: *

# Crawl-Delay: 2 .. (non-standard, but some use it)

Disallow: /files/
Disallow: /calendar/
Disallow: /contact/
Disallow: /credits/
Disallow: /cv/
Disallow: /images/
Disallow: /logs/
Disallow: /mail/
Disallow: /media/
Disallow: /offline/
Disallow: /sitemap/
Disallow: /style/

# Note to myself -
#
# Allow: .... (non-standard, but some use it)
#