- added a missing entry in YaCy interface robots.txt for bookmarks

- changed default robots.txt deny list to include some more interface pages because the loading of such pages are a peer load issue for YaCy when crawlers come by and information on these pages are not useful for public search. 

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7112 6c8d7289-2bf4-0310-a012-ef5d649a1542

defaults/yacy.init

@@ -879,7 +879,7 @@ thumbnailProgram =
 # - status    : peer's status page
 # - surftips  : the surftips-page
 # - wiki      : the wiki-page
-httpd.robots.txt = locked,dirs
+httpd.robots.txt = locked,dirs,bookmarks,network,news,status,profile
 
 # class to use for parsing wikicode
 wikiParser.class = de.anomic.data.wikiCode

htroot/robots.txt

@@ -15,6 +15,10 @@ Disallow: /#[dir]#
 Disallow: /#[page]#
 #{/locked}#
 
+#(bookmarks)#::
+# bookmarks
+Disallow: /Bookmarks.html#(/bookmarks)#
+
 #(wiki)#::
 # wiki
 Disallow: /Wiki.html#(/wiki)#