# RedFM Webapp Robots.txt
# This file tells search engines which pages they can and cannot crawl

User-agent: *
Allow: /

# Allow crawling of main content pages
Allow: /audio/
Allow: /video/
Allow: /events/
Allow: /redlines/
Allow: /talenthunt/
Allow: /rjs/

# Allow crawling of static assets
Allow: /assets/
Allow: /images/
Allow: /fonts/
Allow: /favicon.ico

# Allow crawling of public API endpoints (if any)
Allow: /api/public/
Allow: /api/content/

# Disallow crawling of sensitive areas
Disallow: /auth/
Disallow: /admin/
Disallow: /api/private/
Disallow: /api/admin/
Disallow: /api/user/
Disallow: /api/auth/

# Disallow crawling of development and build files
Disallow: /src/
Disallow: /node_modules/
Disallow: /dist/
Disallow: /.git/
Disallow: /k8s/
Disallow: /nginx/

# Disallow crawling of configuration files
Disallow: /*.json$
Disallow: /*.config.js$
Disallow: /*.config.ts$
Disallow: /tsconfig*
Disallow: /vite.config*
Disallow: /tailwind.config*
Disallow: /postcss.config*
Disallow: /eslint.config*
Disallow: /package.json
Disallow: /bun.lockb
Disallow: /Dockerfile
Disallow: /.nvmrc
Disallow: /.gitignore
Disallow: /README.md

# Disallow crawling of temporary and cache files
Disallow: /temp/
Disallow: /cache/
Disallow: /.vite/
Disallow: /coverage/
Disallow: /logs/

# Disallow crawling of user-specific content
Disallow: /profile/
Disallow: /settings/
Disallow: /dashboard/
Disallow: /account/

# Allow crawling of sitemap (if you have one)
Allow: /sitemap.xml
Allow: /sitemap-index.xml

# Crawl delay for respectful crawling (optional)
Crawl-delay: 1

# Sitemap location (uncomment and update if you have a sitemap)
# Sitemap: https://your-domain.com/sitemap.xml

# Specific rules for major search engines

# Google
User-agent: Googlebot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/private/
Disallow: /api/admin/
Disallow: /api/user/
Disallow: /api/auth/

# Bing
User-agent: Bingbot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/private/
Disallow: /api/admin/
Disallow: /api/user/
Disallow: /api/auth/

# Facebook
User-agent: facebookexternalhit
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/private/
Disallow: /api/admin/
Disallow: /api/user/
Disallow: /api/auth/

# Twitter
User-agent: Twitterbot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/private/
Disallow: /api/admin/
Disallow: /api/user/
Disallow: /api/auth/

# LinkedIn
User-agent: LinkedInBot
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/private/
Disallow: /api/admin/
Disallow: /api/user/
Disallow: /api/auth/

# WhatsApp
User-agent: WhatsApp
Allow: /
Disallow: /auth/
Disallow: /admin/
Disallow: /api/private/
Disallow: /api/admin/
Disallow: /api/user/
Disallow: /api/auth/

# Block bad bots
User-agent: *
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Baiduspider
Disallow: /

# Allow all other bots
User-agent: *
Allow: /