# robots.txt for misovalko.github.io
# Guide search engine crawlers

User-agent: *
Allow: /

# Disallow crawling of backup and temporary files
Disallow: /website-automation/
Disallow: /scripts/
Disallow: /data/
Disallow: /tmp/
Disallow: /cv/*.log
Disallow: /cv/*.aux
Disallow: /cv/*.out
Disallow: /*.bak$
Disallow: /*~$

# Disallow old/deprecated legacy pages
Disallow: /camera.html
Disallow: /comps.html
Disallow: /fmfi-grades.html
Disallow: /fmfi-timetable.html
Disallow: /fmfi-taken-courses.html
Disallow: /pitt-taken-courses.html
Disallow: /phd-calls.html
Disallow: /phdthesis.html
Disallow: /project-*.html
Disallow: /ta-cs0007-fall2005.html
Disallow: /splash.html
Disallow: /misc.html
Disallow: /musicold.html
Disallow: /oktava2000.html
Disallow: /lisboa.html
Disallow: /glee.html
Disallow: /plasyn.html
Disallow: /soc-inet.html

# Disallow old course materials and archives
Disallow: /projects/courses/
Disallow: /projects/phd_comprehensive_exam/
Disallow: /projects/seminar/
Disallow: /projects/fmfi/
Disallow: /projects/skms/
Disallow: /projects/sav/
Disallow: /projects/stosoo/
Disallow: /projects/ties/

# Allow access to main content
Allow: /images/
Allow: /publications/
Allow: /projects/bandits/
Allow: /projects/mva/

# Allow new content pages
Allow: /collaborate.html
Allow: /coauthors.html

# Allow feeds for discovery
Allow: /feed.xml
Allow: /notes-feed.xml

# Crawl delay for polite crawling
Crawl-delay: 1

# Sitemap locations
Sitemap: https://misovalko.github.io/sitemap.xml
Sitemap: https://misovalko.github.io/sitemap-images.xml