feat: 🔍 Block all AI crawlers in robots.txt, make users who opt out of indexing not-indexable

assets/robots.txt

@@ -0,0 +1,24 @@
+User-agent: AdsBot-Google
+User-agent: Amazonbot
+User-agent: anthropic-ai
+User-agent: Applebot-Extended
+User-agent: Bytespider
+User-agent: CCBot
+User-agent: ChatGPT-User
+User-agent: ClaudeBot
+User-agent: Claude-Web
+User-agent: cohere-ai
+User-agent: Diffbot
+User-agent: FacebookBot
+User-agent: FriendlyCrawler
+User-agent: Google-Extended
+User-agent: GoogleOther
+User-agent: GPTBot
+User-agent: img2dataset
+User-agent: omgili
+User-agent: omgilibot
+User-agent: peer39_crawler
+User-agent: peer39_crawler/1.0
+User-agent: PerplexityBot
+User-agent: YouBot
+Disallow: /

nuxt.config.ts

@@ -597,6 +597,9 @@ export default defineNuxtConfig({
     ogImage: {
         enabled: false,
     },
+    robots: {
+        blockNonSeoBots: true,
+    },
     veeValidate: {
         autoImports: true,
         componentNames: {

pages/[username]/[uuid].vue

@@ -54,5 +54,11 @@ useServerSeoMeta({
     title: note.value?.account.display_name,
     description: note.value?.content,
     ogImage: note.value?.media_attachments[0]?.preview_url,
+    robots: computed(() => ({
+        noindex: !!note.value?.account.noindex,
+        nofollow: !!note.value?.account.noindex,
+        noarchive: !!note.value?.account.noindex,
+        noimageindex: !!note.value?.account.noindex,
+    })),
 });
 </script>

pages/[username]/index.vue

@@ -56,5 +56,11 @@ useServerSeoMeta({
     description: computed(() =>
         account.value ? account.value.note : undefined,
     ),
+    robots: computed(() => ({
+        noindex: !!account.value?.noindex,
+        nofollow: !!account.value?.noindex,
+        noarchive: !!account.value?.noindex,
+        noimageindex: !!account.value?.noindex,
+    })),
 });
 </script>