From badd07e40c80bcf8252e20b2df87af0f197f3ce1 Mon Sep 17 00:00:00 2001 From: yctct Date: Wed, 22 Apr 2026 16:44:37 +0200 Subject: First commit, add files --- robots.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 robots.txt (limited to 'robots.txt') diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..0868b21 --- /dev/null +++ b/robots.txt @@ -0,0 +1,25 @@ +User-agent: * # all bots +Disallow: # allow this directory (the entire website) + +User-agent: googlebot # all Google services +Disallow: / # disallow this directory (the entire website) + +User-agent: GPTBot # ChatGPT +Disallow: / # disallow this directory (the entire website) + +User-agent: Bytespider +Disallow: / + +# on its website, Google stipulate that instructions listed in robots.txt +# file are not enough to blocking Google from indexing a webiste: +# 'it is not a mechanism for keeping a web page out of Google.' +# +# if you want to block all search indexing +# you need to add the following header to all of your pages: +# +# although Google stipulates that '[f]or the noindex rule to be effective, +# the page or resource must not be blocked by a robots.txt file' +# sound like a catch-22. idk. + + + -- cgit v1.2.3