# Sitemap location (Ensure this is correct for your domain)
Sitemap: https://espressothoughts.com/sitemap.xml

# Allow all search engines to crawl the site
User-agent: *
Disallow: /admin/
Disallow: /login/
Disallow: /register/
Disallow: /cart/
Disallow: /checkout/
Disallow: /search?q=*
Disallow: /private/
Disallow: /tmp/
Disallow: /api/

# Allow important pages even if inside restricted folders
Allow: /public/
Allow: /article/
Allow: /poem/

# Block specific bots that may cause excessive load
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Prevent search engines from indexing duplicate content
Disallow: /tag/
Disallow: /categories/

# Prevent indexing of dynamically generated pages
Disallow: /*?replytocom
Disallow: /*?utm_source
Disallow: /*?fbclid

# Avoid indexing of specific file types
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /*.env$
Disallow: /*.css$
Disallow: /*.js$