diff --git a/README.md b/README.md index 42af4aa9e6..aad5b4d8b3 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ Please don't use ElasticSearch unless you already have an ElasticSearch setup an - To add custom error images, place them in the `./custom/assets/badges` directory, replacing the files already there. - To add custom sounds, place only mp3 files in the `./custom/assets/sounds` directory. - To update custom assets without rebuilding, just run `pnpm run gulp`. +- To block ChatGPT, CommonCrawl, or other crawlers from indexing your instance, uncomment the respective rules in `./custom/robots.txt`. ## 🧑‍🔬 Configuring a new server diff --git a/custom/assets/robots.txt b/custom/assets/robots.txt new file mode 100644 index 0000000000..cedb5b6cf2 --- /dev/null +++ b/custom/assets/robots.txt @@ -0,0 +1,14 @@ +User-agent: * +Allow: / + +# Uncomment the following to block CommonCrawl +# +# User-agent: CCBot +# User-agent: CCBot/2.0 +# User-agent: CCBot/3.1 +# Disallow: / + +# Uncomment the following to block ChatGPT +# +# User-agent: GPTBot +# Disallow: / diff --git a/packages/backend/src/server/web/views/clip.pug b/packages/backend/src/server/web/views/clip.pug index 79d6293808..b3285dbe2e 100644 --- a/packages/backend/src/server/web/views/clip.pug +++ b/packages/backend/src/server/web/views/clip.pug @@ -24,9 +24,11 @@ block meta unless privateMode if profile.noCrawle meta(name='robots' content='noindex') + if profile.preventAiLearning meta(name='robots' content='noai') meta(name='robots' content='noimageai') + meta(name='GPTBot' content='noindex') meta(name='misskey:user-username' content=user.username) meta(name='misskey:user-id' content=user.id)