# Please do not crawl us faster than 1 hit/second.
#
# If you need to examine many dmoz pages, please download the rdf file from
# http://rdf.dmoz.org/ instead of crawling us.
#
User-agent: *
Crawl-Delay: 1
Allow: /public/profile 
Disallow: /public/
Disallow: /editors/
Disallow: /search/
Disallow: /search
Disallow: /cgi-bin/