# Gorka - Robots.txt
# Last Updated: 2025-12-29

User-agent: *
Allow: /

# Disallow admin areas
Disallow: /admin/
Disallow: /api/

# Disallow private/owned token pages
Disallow: /token/owned/
Disallow: /iteration/owned/
Disallow: /objkt/token/owned/
Disallow: /fxhash/iteration/owned/

# Allow specific API endpoints for SEO
Allow: /api/sitemap.xml
Allow: /api/sitemap-index.xml
Allow: /api/sitemap-collections.xml
Allow: /api/sitemap-projects.xml
Allow: /api/robots.txt
Allow: /api/feed.xml
Allow: /api/manifest.json
Allow: /api/opensearch.xml
Allow: /api/favicon/

# Crawl delay for politeness
Crawl-delay: 1

# Sitemap index (references all sitemaps)
Sitemap: naturefrequency.art/api/sitemap-index.xml

# Individual sitemaps for faster discovery
Sitemap: naturefrequency.art/api/sitemap.xml
Sitemap: naturefrequency.art/api/sitemap-collections.xml
Sitemap: naturefrequency.art/api/sitemap-projects.xml

# Search engine specific rules
User-agent: Googlebot
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Crawl-delay: 1

User-agent: Slurp
Crawl-delay: 2

User-agent: DuckDuckBot
Crawl-delay: 1

# Social media crawlers (always allow for rich previews)
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0

User-agent: Twitterbot
Allow: /
Crawl-delay: 0

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Slackbot
Allow: /

# AI crawlers (respect preferences)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Disallow: /