User-agent: *
Content-Signal: search=yes,ai-train=yes,ai-input=yes,use=full
Allow: /

# Allow all search engines to crawl the directory
Allow: /directory/
Allow: /directory/prompt/
Allow: /directory/submit/
Allow: /directory/category/

# Block admin and private areas
Disallow: /admin/
Disallow: /admin-dashboard/
Disallow: /directory/admin/
Disallow: /directory/api/
Disallow: /authentication/
Disallow: /accounts/
Disallow: /accounts/login/
Disallow: /login/
Disallow: /register/
Disallow: /password-reset/
Disallow: /logout/
# Same pages behind a language prefix (/de/accounts/google/login/ etc.).
# Without these, crawlers fetched the login page ~9k times a day.
Disallow: /*/accounts/
Disallow: /*/login/
Disallow: /*/register/
Disallow: /*/password-reset/
Disallow: /*/logout/

# CRITICAL: Block ALL user profile pages from indexing
Disallow: /user/
Disallow: /users/
Disallow: /profile/
Disallow: /profiles/
Disallow: /u/
Disallow: /@*/
Disallow: /member/
Disallow: /members/

# Block search and filter URLs to prevent duplicate content
Disallow: /*?search=
Disallow: /*?sort=
Disallow: /*?difficulty=
Disallow: /*?page=
Disallow: /*?filter=
Disallow: /*?tab=

# Allow CSS, JS, and images
Allow: /static/
Allow: *.css
Allow: *.js
Allow: *.png
Allow: *.jpg
Allow: *.jpeg
Allow: *.gif
Allow: *.svg
Allow: *.webp

# Sitemap location
Sitemap: https://ulazai.com/sitemap.xml

# Crawl delay (be nice to servers)
Crawl-delay: 1

# Specific instructions for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: Slurp
Allow: /
Crawl-delay: 2