# itinypdf.com robots.txt

User-agent: *
Allow: /

# Block non-content directories
Disallow: /api/
Disallow: /admin/
Disallow: /src/

# Block UTM and referral parameter URLs (prevents duplicate pages in GSC)
Disallow: /*?ref=
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?utm_term=
Disallow: /*?utm_content=

# Block service worker and config files
Disallow: /sw.js

# Allow crawling of main assets (helps Google render pages for CWV)
Allow: /assets/
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.png$
Allow: /*.svg$
Allow: /*.webp$

Sitemap: https://itinypdf.com/sitemap.xml

# ─── Search engine bots ───────────────────────────────────────────

User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# ─── AI bots ──────────────────────────────────────────────────────

User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

User-agent: CCBot
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Crawl-delay: 2

User-agent: Claude-Web
Allow: /
Crawl-delay: 2