User-agent: *
Allow: /

Sitemap: https://frame42.ai/sitemap.xml

# Disable crawling of unnecessary files
Disallow: /css/
Disallow: /js/
Disallow: /includes/
Disallow: /assets/
Disallow: /vendor/

# Allow all other user agents to all other content
User-Agent: *
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.mp4$
Allow: /*.webm$

# Crawl-delay: 1

# Block specific pages if needed
# Disallow: /private/
# Disallow: /admin/