First commit, add files

1 files changed, 25 insertions, 0 deletions

diff --git a/robots.txt b/robots.txt
new file mode 100644
index 0000000..0868b21
--- /dev/null
+++ b/robots.txt
@@ -0,0 +1,25 @@
+User-agent: *			# all bots			 
+Disallow:           		# allow this directory (the entire website)
+
+User-agent: googlebot        	# all Google services
+Disallow: /          		# disallow this directory (the entire website)
+
+User-agent: GPTBot		# ChatGPT
+Disallow: /          		# disallow this directory (the entire website)
+
+User-agent: Bytespider
+Disallow: /
+
+# on its website, Google stipulate that instructions listed in robots.txt
+# file are not enough to blocking Google from indexing a webiste:  
+# 'it is not a mechanism for keeping a web page out of Google.'
+#
+# if you want to block all search indexing 
+# you need to add the following header to all of your pages:
+# <meta name="robots" content="noindex" />
+# although Google stipulates that '[f]or the noindex rule to be effective, 
+# the page or resource must not be blocked by a robots.txt file'
+# sound like a catch-22. idk.
+
+
+