User-agent: *
Allow: /

# Block URL parameters to prevent duplicate content
Disallow: /*?lang=
Disallow: /*?q=
Disallow: /*?search_term_string

Crawl-delay: 1

# Specific directives for major search engines
User-agent: Googlebot
Allow: /
Disallow: /*?lang=
Disallow: /*?q=
Disallow: /*?search_term_string
# Note: Google ignores Crawl-delay

User-agent: Googlebot-Image
Allow: /

# Google AdSense crawler - CRITICAL for ad approval
# Must explicitly allow query params that are blocked for other crawlers
User-agent: Mediapartners-Google
Allow: /*?lang=
Allow: /*?q=
Allow: /

User-agent: AdsBot-Google
Allow: /*?lang=
Allow: /*?q=
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /*?lang=
Allow: /*?q=
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Social media bots for sharing
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

# Block AI scrapers to prevent content theft
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: PerplexityBot
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Omgili
Allow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Bytespider
Disallow: /

# Sitemap locations (sitemap-index.xml references all sitemaps)
Sitemap: https://pdfcompressorunlimited.com/sitemap-index.xml
Sitemap: https://pdfcompressorunlimited.com/sitemap.xml
Sitemap: https://pdfcompressorunlimited.com/image-sitemap.xml