From 0ab6a462ee4bdca684aa24b19872f2fcc1f53b34 Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Mon, 6 Sep 2010 09:58:54 +0000
Subject: [PATCH] - added a missing entry in YaCy interface robots.txt for
 bookmarks - changed default robots.txt deny list to include some more
 interface pages because the loading of such pages are a peer load issue for
 YaCy when crawlers come by and information on these pages are not useful for
 public search.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7112 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 defaults/yacy.init | 2 +-
 htroot/robots.txt  | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/defaults/yacy.init b/defaults/yacy.init
index ea8f4bd94..8664f0d6e 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -879,7 +879,7 @@ thumbnailProgram =
 # - status    : peer's status page
 # - surftips  : the surftips-page
 # - wiki      : the wiki-page
-httpd.robots.txt = locked,dirs
+httpd.robots.txt = locked,dirs,bookmarks,network,news,status,profile
 
 # class to use for parsing wikicode
 wikiParser.class = de.anomic.data.wikiCode
diff --git a/htroot/robots.txt b/htroot/robots.txt
index cfb7d3567..3ae5a593e 100644
--- a/htroot/robots.txt
+++ b/htroot/robots.txt
@@ -15,6 +15,10 @@ Disallow: /#[dir]#
 Disallow: /#[page]#
 #{/locked}#
 
+#(bookmarks)#::
+# bookmarks
+Disallow: /Bookmarks.html#(/bookmarks)#
+
 #(wiki)#::
 # wiki
 Disallow: /Wiki.html#(/wiki)#