# desitube.xyz robots
User-agent: *
Allow: /
Allow: /category/
Allow: /tag/
Allow: /api/video/proxy
Allow: /api/video/frame
Disallow: /api/
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?site=
Disallow: /*&site=
Disallow: /*?q=
Disallow: /*&q=
Disallow: /random

# Search crawlers read only their own group (never *) — repeat the full policy.
User-agent: Googlebot
Allow: /
Allow: /category/
Allow: /tag/
Allow: /api/video/proxy
Allow: /api/video/frame
Disallow: /api/
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?site=
Disallow: /*&site=
Disallow: /*?q=
Disallow: /*&q=
Disallow: /random

User-agent: Googlebot-Image
Allow: /
Allow: /category/
Allow: /tag/
Allow: /api/video/proxy
Allow: /api/video/frame
Disallow: /api/
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?site=
Disallow: /*&site=
Disallow: /*?q=
Disallow: /*&q=
Disallow: /random

User-agent: Bingbot
Allow: /
Allow: /category/
Allow: /tag/
Allow: /api/video/proxy
Allow: /api/video/frame
Disallow: /api/
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?site=
Disallow: /*&site=
Disallow: /*?q=
Disallow: /*&q=
Disallow: /random

User-agent: Yandex
Allow: /
Allow: /category/
Allow: /tag/
Allow: /api/video/proxy
Allow: /api/video/frame
Disallow: /api/
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?site=
Disallow: /*&site=
Disallow: /*?q=
Disallow: /*&q=
Disallow: /random

User-agent: DuckDuckBot
Allow: /
Allow: /category/
Allow: /tag/
Allow: /api/video/proxy
Allow: /api/video/frame
Disallow: /api/
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?site=
Disallow: /*&site=
Disallow: /*?q=
Disallow: /*&q=
Disallow: /random

# Social/preview crawlers: keep permissive so OG/Twitter-card fetches never break.
User-agent: Twitterbot
Allow: /
User-agent: facebookexternalhit
Allow: /

Sitemap: https://desitube.xyz/sitemap-pages.xml
Sitemap: https://desitube.xyz/sitemap-videos.xml