User-agent: *
Allow: /
Disallow: /api/
Disallow: /en/company/download/prompt*
Disallow: /fr/company/download/prompt*
Disallow: /de/company/download/prompt*
Disallow: /en/analytics/*
Disallow: /fr/analytics/*
Disallow: /de/analytics/*

# Allow clean SEO URLs for companies pages
Allow: /en/companies/market/
Allow: /en/companies/sector/
Allow: /en/companies/marketcap/
Allow: /fr/companies/market/
Allow: /fr/companies/sector/
Allow: /fr/companies/marketcap/
Allow: /de/companies/market/
Allow: /de/companies/sector/
Allow: /de/companies/marketcap/

# Block paginated versions of filtered results (let canonical handle it)
Disallow: /en/companies?*page=
Disallow: /fr/companies?*page=
Disallow: /de/companies?*page=

# Block sort variations (duplicate content)
Disallow: /en/companies?*sort_by=
Disallow: /fr/companies?*sort_by=
Disallow: /de/companies?*sort_by=
Disallow: /en/companies?*sort_order=
Disallow: /fr/companies?*sort_order=
Disallow: /de/companies?*sort_order=

# Block lookback variations (same data, different time window)
Disallow: /en/companies?*lookback=
Disallow: /fr/companies?*lookback=
Disallow: /de/companies?*lookback=

# Rate limiting for crawlers (2 seconds allows efficient indexing while being server-friendly)
Crawl-delay: 2

Sitemap: https://www.insiderscreener.com/sitemap.xml