# robots.txt for https://law.duke.edu

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /misc/*.css$
Allow: /misc/*.css?
Allow: /misc/*.js$
Allow: /misc/*.js?
Allow: /misc/*.gif
Allow: /misc/*.jpg
Allow: /misc/*.jpeg
Allow: /misc/*.png
Allow: /modules/*.css$
Allow: /modules/*.css?
Allow: /modules/*.js$
Allow: /modules/*.js?
Allow: /modules/*.gif
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.png
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /themes/*.css$
Allow: /themes/*.css?
Allow: /themes/*.js$
Allow: /themes/*.js?
Allow: /themes/*.gif
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.png
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
# ETC
Disallow: /taxonomy/
Disallow: /taxonomy/term/
Disallow: /fac/taxonomy
Disallow: /boylesite/
Disallow: /plasma/
Disallow: /admis/mordecaiupdate/
Disallow: /admis/secure/
Disallow: /aidsproject/400_01/secure/
Disallow: /alumni/charts/
Disallow: /alumni/involve/leadership/
Disallow: /alumni/notes/
Disallow: /apps/
Disallow: /career/alumops/
Disallow: /career/ALUMOPS/
Disallow: /career/calendar/
Disallow: /career/calendarprotected/
Disallow: /career/password/
Disallow: /career/password/clerkships/
Disallow: /career/profdev/password/
Disallow: /career/toolkit/
Disallow: /cgi-shl/
Disallow: /commphotos/
Disallow: /communityinfo/ppsurvey_files/
Disallow: /communityinfo/secure/
Disallow: /communityinfo/herald/archives/
Disallow: /conference/1999/
Disallow: /conference/2000/
Disallow: /conference/2001/
Disallow: /conference/2002/
Disallow: /conference/2003/
Disallow: /conference/2004/
Disallow: /conference/2005/
Disallow: /conference/2006/
Disallow: /conference/2007/
Disallow: /conference/2008/
Disallow: /conference/2009/
Disallow: /conference/2010/
Disallow: /conference/2011/
Disallow: /content/
Disallow: /cpr/secure/
Disallow: /cspd/secure/
Disallow: /curriculum/deprecated_courseHomepages/
Disallow: /curriculum/deprecated_courseHomepages/130_04/
Disallow: /curriculum/deprecated_courseHomepages/275_01/
Disallow: /curriculum/deprecated_courseHomepages/508_01/secure/
Disallow: /curriculum/deprecated_courseHomepages/544_01/
Disallow: /curriculum/deprecated_courseHomepages/Fall2002/227_01/secure/
Disallow: /curriculum/deprecated_courseHomepages/Fall2002/275_01/secure/
Disallow: /curriculum/deprecated_courseHomepages/Fall2002/351_01/secure/
Disallow: /curriculum/deprecated_courseHomepages/Spring2002/448_01/secure/
Disallow: /curriculum/deprecated_courseHomepages/Spring2002/509_01/secure/
Disallow: /curriculum/deprecated_courseHomepages/protected/
Disallow: /curriculum/ematerials/
Disallow: /curriculum/studentscholar/
Disallow: /distinct/
Disallow: /durham/video/
Disallow: /fac/benjamin/telecom/teach/
Disallow: /fac/benjamin/telecom/teach2/
Disallow: /fac/benjamin/telecom/teach3/
Disallow: /facultysupport/manual/
Disallow: /facpub/scholarship/
Disallow: /history/faculty/
Disallow: /infoweb/cilpgrab.php
Disallow: /infoweb/various/tally.php
Disallow: /innocencecenter/password_protected/
Disallow: /internat/admitted/home/
Disallow: /internat/admitted/health/
Disallow: /internat/admitted/housing/
Disallow: /internat/admitted/courses/
Disallow: /internat/admitted/usbar/
Disallow: /internat/admitted/courses/
Disallow: /internat/admitted/computer/
Disallow: /internat/admitted/career/
Disallow: /internat/admitted/tuition/
Disallow: /internat/admitted/arrival/
Disallow: /internat/admitted/driving/
Disallow: /internat/admitted/tips/
Disallow: /internat/admitted/maps/
Disallow: /internat/admitted/legalsystem/
Disallow: /internat/admitted/contact/
Disallow: /intranet/
Disallow: /js/
Disallow: /lib/CILP/
Disallow: /lib/cilpgrab/
Disallow: /lib/INTERNAL/
Disallow: /lib/internal/
Disallow: /lib/lawref/
Disallow: /lib/new/
Disallow: /llm2003/
Disallow: /newhistory/
Disallow: /news/duke-center-sports-and-law-host-panel-discussion-among-high-level-sports-executives-1/
Disallow: /phpclasses/
Disallow: /publiclaw/conference/epa/
Disallow: /publiclaw/conference/exileconference/
Disallow: /publiclaw/conference/fall2001/
Disallow: /publiclaw/conference/fall2002/
Disallow: /publiclaw/conference/fall2004/
Disallow: /ramgen/
Disallow: /restricteddownloads/
Disallow: /scholarship/dukelawresearch/
Disallow: /scholarship/facact/
Disallow: /sites/default/files/migrated_files/
Disallow: /slide/
Disallow: /staffphotos/restricted/
Disallow: /student/act/BLACKLSA/password/
Disallow: /student/act/DBA/survey/
Disallow: /student/act/DBA_CSB/secure/
Disallow: /student/act/innocence/secure/
Disallow: /student/act/mootCourt/law_secure/
Disallow: /student/act/mootCourt/secure/
Disallow: /student/act/transfer/secure/
Disallow: /student/act/wlsa/password/
Disallow: /studentaffairs/disabilities/
Disallow: /temp/
Disallow: /test/
Disallow: /trips/papers/
Disallow: /utils/
Disallow: /webcast/kennedy/
Disallow: /webcast/secure/
Disallow: /welcome/home/
Disallow: /welcome/enrolling/
Disallow: /welcome/transfer/
Disallow: /welcome/durham/
Disallow: /welcome/durham/shopping/
Disallow: /welcome/durham/culture/
Disallow: /welcome/durham/sports/
Disallow: /welcome/durham/housing/
Disallow: /welcome/budget/
Disallow: /welcome/budget/tuition/
Disallow: /welcome/budget/studentexpense/
Disallow: /welcome/budget/applying/
Disallow: /welcome/budget/scholarship/
Disallow: /welcome/budget/loans/
Disallow: /welcome/budget/letters/
Disallow: /welcome/budget/aid/
Disallow: /welcome/contact/
Disallow: /welcome/summer/
Disallow: /lib/coursepack/

User-agent: 008
Disallow: /

User-agent: daumoa
Disallow: /

User-agent: Baiduspider
User-agent: baiduspider
User-agent: Baiduspider+ 
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Crawl-delay: 10