From 6c43374d8eec3546c420f91168b1747eed3d4f0e Mon Sep 17 00:00:00 2001
From: Jesse Wierzbinski <contact@cpluspatch.com>
Date: Thu, 20 Jun 2024 20:01:33 -1000
Subject: [PATCH] feat: :mag: Block all AI crawlers in robots.txt, make users
 who opt out of indexing not-indexable

---
 assets/robots.txt           | 24 ++++++++++++++++++++++++
 nuxt.config.ts              |  3 +++
 pages/[username]/[uuid].vue |  6 ++++++
 pages/[username]/index.vue  |  6 ++++++
 4 files changed, 39 insertions(+)
 create mode 100644 assets/robots.txt

diff --git a/assets/robots.txt b/assets/robots.txt
new file mode 100644
index 0000000..aa34805
--- /dev/null
+++ b/assets/robots.txt
@@ -0,0 +1,24 @@
+User-agent: AdsBot-Google
+User-agent: Amazonbot
+User-agent: anthropic-ai
+User-agent: Applebot-Extended
+User-agent: Bytespider
+User-agent: CCBot
+User-agent: ChatGPT-User
+User-agent: ClaudeBot
+User-agent: Claude-Web
+User-agent: cohere-ai
+User-agent: Diffbot
+User-agent: FacebookBot
+User-agent: FriendlyCrawler
+User-agent: Google-Extended
+User-agent: GoogleOther
+User-agent: GPTBot
+User-agent: img2dataset
+User-agent: omgili
+User-agent: omgilibot
+User-agent: peer39_crawler
+User-agent: peer39_crawler/1.0
+User-agent: PerplexityBot
+User-agent: YouBot
+Disallow: /
\ No newline at end of file
diff --git a/nuxt.config.ts b/nuxt.config.ts
index 8589e67..b80d418 100644
--- a/nuxt.config.ts
+++ b/nuxt.config.ts
@@ -597,6 +597,9 @@ export default defineNuxtConfig({
     ogImage: {
         enabled: false,
     },
+    robots: {
+        blockNonSeoBots: true,
+    },
     veeValidate: {
         autoImports: true,
         componentNames: {
diff --git a/pages/[username]/[uuid].vue b/pages/[username]/[uuid].vue
index 8ae1d29..687be10 100644
--- a/pages/[username]/[uuid].vue
+++ b/pages/[username]/[uuid].vue
@@ -54,5 +54,11 @@ useServerSeoMeta({
     title: note.value?.account.display_name,
     description: note.value?.content,
     ogImage: note.value?.media_attachments[0]?.preview_url,
+    robots: computed(() => ({
+        noindex: !!note.value?.account.noindex,
+        nofollow: !!note.value?.account.noindex,
+        noarchive: !!note.value?.account.noindex,
+        noimageindex: !!note.value?.account.noindex,
+    })),
 });
 </script>
\ No newline at end of file
diff --git a/pages/[username]/index.vue b/pages/[username]/index.vue
index fabe1ea..e822aa8 100644
--- a/pages/[username]/index.vue
+++ b/pages/[username]/index.vue
@@ -56,5 +56,11 @@ useServerSeoMeta({
     description: computed(() =>
         account.value ? account.value.note : undefined,
     ),
+    robots: computed(() => ({
+        noindex: !!account.value?.noindex,
+        nofollow: !!account.value?.noindex,
+        noarchive: !!account.value?.noindex,
+        noimageindex: !!account.value?.noindex,
+    })),
 });
 </script>
\ No newline at end of file