# All robots will spider the domain
User-agent: *
Disallow: /database/
Disallow: /bin/
Disallow: /css/
Disallow: /flash/
Disallow: /images/
Disallow: /admin/
Disallow: /video/
Disallow: /art/
Disallow: /templates/
Disallow: /content/
Disallow: /stats/