User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/
Disallow: /website-editor/
Disallow: /*.json$
Disallow: /*.xml$

# Crawl-delay for all bots
Crawl-delay: 1

# Sitemap location
Sitemap: https://lovemedia.org.za/sitemap.xml

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/
Disallow: /website-editor/

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /dashboard/
Disallow: /website-editor/

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# Block unwanted bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /