- robots.txt is a servlet now - no need to rewrite the whole file each time a section is added or removed - user-defined disallows, added manually, won't be overwritten anymore - new config-setting: httpd.robots.txt, holding names of the disallowed sections git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3423 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
karlchenofhell
18 years ago
parent
9623bf7bbe
commit
88245e44d8
@ -0,0 +1,71 @@
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import de.anomic.http.httpHeader;
|
||||
import de.anomic.http.httpdRobotsTxtConfig;
|
||||
import de.anomic.plasma.plasmaSwitchboard;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
import de.anomic.server.servletProperties;
|
||||
|
||||
public class robots {
|
||||
|
||||
public static servletProperties respond(httpHeader header, serverObjects post, serverSwitch env) {
|
||||
final servletProperties prop = new servletProperties();
|
||||
final httpdRobotsTxtConfig rbc = ((plasmaSwitchboard)env).robotstxtConfig;
|
||||
|
||||
if (rbc.isAllDisallowed()) {
|
||||
prop.put(httpdRobotsTxtConfig.ALL, 1);
|
||||
} else {
|
||||
if (rbc.isBlogDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.BLOG, 1);
|
||||
if (rbc.isBookmarksDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.BOOKMARKS, 1);
|
||||
if (rbc.isFileshareDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.FILESHARE, 1);
|
||||
if (rbc.isHomepageDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.HOMEPAGE, 1);
|
||||
if (rbc.isNetworkDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.NETWORK, 1);
|
||||
if (rbc.isNewsDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.NEWS, 1);
|
||||
if (rbc.isStatusDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.STATUS, 1);
|
||||
if (rbc.isSurftipsDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.SURFTIPS, 1);
|
||||
if (rbc.isWikiDisallowed()) prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.WIKI, 1);
|
||||
|
||||
if (rbc.isLockedDisallowed() || rbc.isDirsDisallowed()) {
|
||||
final ArrayList[] p = getFiles(env.getConfig(plasmaSwitchboard.HTROOT_PATH, plasmaSwitchboard.HTROOT_PATH_DEFAULT));
|
||||
if (rbc.isLockedDisallowed()) {
|
||||
prop.put(httpdRobotsTxtConfig.LOCKED, p[0].size());
|
||||
for (int i=0; i<p[0].size(); i++)
|
||||
prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.LOCKED + "_" + i + "_page", p[0].get(i));
|
||||
}
|
||||
if (rbc.isDirsDisallowed()) {
|
||||
prop.put(httpdRobotsTxtConfig.DIRS, p[1].size());
|
||||
for (int i=0; i<p[1].size(); i++)
|
||||
prop.put(httpdRobotsTxtConfig.ALL + "_" + httpdRobotsTxtConfig.DIRS + "_" + i + "_dir", p[1].get(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return prop;
|
||||
}
|
||||
|
||||
private static ArrayList[] getFiles(String htrootPath) {
|
||||
final File htroot = new File(htrootPath);
|
||||
if (!htroot.exists()) return null;
|
||||
final ArrayList htrootFiles = new ArrayList();
|
||||
final ArrayList htrootDirs = new ArrayList();
|
||||
final String[] htroots = htroot.list();
|
||||
File file;
|
||||
for (int i=0, dot; i<htroots.length; i++) {
|
||||
if (htroots[i].equals("www")) continue;
|
||||
file = new File(htroot, htroots[i]);
|
||||
if (file.isDirectory()) {
|
||||
htrootDirs.add("/" + file.getName());
|
||||
} else if (
|
||||
(dot = htroots[i].lastIndexOf('.')) < 2 ||
|
||||
htroots[i].charAt(dot - 2) == '_' && htroots[i].charAt(dot - 1) == 'p'
|
||||
) {
|
||||
htrootFiles.add("/" + file.getName());
|
||||
}
|
||||
}
|
||||
return new ArrayList[] { htrootFiles, htrootDirs };
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
# robots.txt for #[clientname]#.yacy
|
||||
|
||||
User-agent: *
|
||||
|
||||
#(all)#
|
||||
|
||||
#{dirs}#
|
||||
# dirs
|
||||
Disallow: /#[dir]##{/dirs}#
|
||||
|
||||
#{locked}#
|
||||
# locked
|
||||
Disallow: /#[page]##{/locked}#
|
||||
|
||||
#(wiki)#::
|
||||
# wiki
|
||||
Disallow: /Wiki.html#(/wiki)#
|
||||
|
||||
#(blog)#::
|
||||
# blog
|
||||
Disallow: /Blog.html
|
||||
Disallow: /Blog.rss
|
||||
Disallow: /Blog.xml#(/blog)#
|
||||
|
||||
#(news)#::
|
||||
# news
|
||||
Disallow: /News.html#(/news)#
|
||||
|
||||
#(status)#::
|
||||
# status
|
||||
Disallow: /Status.html#(/status)#
|
||||
|
||||
#(network)#::
|
||||
# network
|
||||
Disallow: /Network.html
|
||||
Disallow: /Network.csv
|
||||
Disallow: /Network.xml#(/network)#
|
||||
|
||||
#(homepage)#::
|
||||
# homepage
|
||||
Disallow: /www#(/homepage)#
|
||||
|
||||
#(fileshare)#::
|
||||
# fileshare
|
||||
Disallow: /share#(/fileshare)#
|
||||
|
||||
#(surftips)#::
|
||||
# surftips
|
||||
Disallow: /Surftips.html#(/surftips)#
|
||||
|
||||
::
|
||||
# all
|
||||
Disallow: /
|
||||
#(/all)#
|
||||
@ -0,0 +1,220 @@
|
||||
// httpdRobotsTxtConfig.java
|
||||
// ---------
|
||||
// part of YaCy
|
||||
// (C) by Michael Peter Christen; mc@anomic.de
|
||||
// first published on http://www.anomic.de
|
||||
// Frankfurt, Germany, 2007
|
||||
// Created 22.02.2007
|
||||
//
|
||||
// This file is contributed by Franz Brauße
|
||||
//
|
||||
// $LastChangedDate: $
|
||||
// $LastChangedRevision: $
|
||||
// $LastChangedBy: $
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
//
|
||||
// Using this software in any meaning (reading, learning, copying, compiling,
|
||||
// running) means that you agree that the Author(s) is (are) not responsible
|
||||
// for cost, loss of data or any harm that may be caused directly or indirectly
|
||||
// by usage of this softare or this documentation. The usage of this software
|
||||
// is on your own risk. The installation and usage (starting/running) of this
|
||||
// software may allow other people or application to access your computer and
|
||||
// any attached devices and is highly dependent on the configuration of the
|
||||
// software which must be done by the user of the software; the author(s) is
|
||||
// (are) also not responsible for proper configuration and usage of the
|
||||
// software, even if provoked by documentation provided together with
|
||||
// the software.
|
||||
//
|
||||
// Any changes to this file according to the GPL as documented in the file
|
||||
// gpl.txt aside this file in the shipment you received can be done to the
|
||||
// lines that follows this copyright notice here, but changes must not be
|
||||
// done inside the copyright notive above. A re-distribution must contain
|
||||
// the intact and unchanged copyright notice.
|
||||
// Contributions and changes to the program code must be marked as such.
|
||||
|
||||
package de.anomic.http;
|
||||
|
||||
import de.anomic.plasma.plasmaSwitchboard;
|
||||
import de.anomic.server.serverSwitch;
|
||||
|
||||
public final class httpdRobotsTxtConfig {
|
||||
|
||||
public static final String WIKI = "wiki";
|
||||
public static final String BLOG = "blog";
|
||||
public static final String BOOKMARKS = "bookmarks";
|
||||
public static final String HOMEPAGE = "homepage";
|
||||
public static final String FILESHARE = "fileshare";
|
||||
public static final String SURFTIPS = "surftips";
|
||||
public static final String NEWS = "news";
|
||||
public static final String STATUS = "status";
|
||||
public static final String LOCKED = "locked";
|
||||
public static final String DIRS = "dirs";
|
||||
public static final String NETWORK = "network";
|
||||
public static final String ALL = "all";
|
||||
|
||||
private boolean allDisallowed = false;
|
||||
private boolean lockedDisallowed = true;
|
||||
private boolean dirsDisallowed = true;
|
||||
private boolean wikiDisallowed = false;
|
||||
private boolean blogDisallowed = false;
|
||||
private boolean fileshareDisallowed = false;
|
||||
private boolean homepageDisallowed = false;
|
||||
private boolean newsDisallowed = false;
|
||||
private boolean statusDisallowed = false;
|
||||
private boolean networkDisallowed = false;
|
||||
private boolean surftipsDisallowed = false;
|
||||
private boolean bookmarksDisallowed = false;
|
||||
|
||||
public httpdRobotsTxtConfig() { }
|
||||
|
||||
public httpdRobotsTxtConfig(String[] active) {
|
||||
if (active == null) return;
|
||||
for (int i=0; i<active.length; i++) {
|
||||
if (active[i] == null) continue;
|
||||
if (active[i].equals(BLOG)) { this.blogDisallowed = true; continue; }
|
||||
if (active[i].equals(WIKI)) { this.wikiDisallowed = true; continue; }
|
||||
if (active[i].equals(BOOKMARKS)) { this.bookmarksDisallowed = true; continue; }
|
||||
if (active[i].equals(HOMEPAGE)) { this.homepageDisallowed = true; continue; }
|
||||
if (active[i].equals(FILESHARE)) { this.fileshareDisallowed = true; continue; }
|
||||
if (active[i].equals(SURFTIPS)) { this.surftipsDisallowed = true; continue; }
|
||||
if (active[i].equals(NEWS)) { this.newsDisallowed = true; continue; }
|
||||
if (active[i].equals(STATUS)) { this.statusDisallowed = true; continue; }
|
||||
if (active[i].equals(NETWORK)) { this.networkDisallowed = true; continue; }
|
||||
if (active[i].equals(LOCKED)) { this.lockedDisallowed = true; continue; }
|
||||
if (active[i].equals(DIRS)) { this.dirsDisallowed = true; continue; }
|
||||
if (active[i].equals(ALL)) { this.allDisallowed = true; continue; }
|
||||
}
|
||||
}
|
||||
|
||||
public static httpdRobotsTxtConfig init(serverSwitch env) {
|
||||
String cfg = env.getConfig(plasmaSwitchboard.ROBOTS_TXT, plasmaSwitchboard.ROBOTS_TXT_DEFAULT);
|
||||
if (cfg == null) return new httpdRobotsTxtConfig();
|
||||
return new httpdRobotsTxtConfig(cfg.split(","));
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
if (this.allDisallowed) return ALL;
|
||||
StringBuffer sb = new StringBuffer();
|
||||
if (this.blogDisallowed) sb.append(BLOG).append(",");
|
||||
if (this.bookmarksDisallowed) sb.append(BOOKMARKS).append(",");
|
||||
if (this.dirsDisallowed) sb.append(DIRS).append(",");
|
||||
if (this.fileshareDisallowed) sb.append(FILESHARE).append(",");
|
||||
if (this.homepageDisallowed) sb.append(HOMEPAGE).append(",");
|
||||
if (this.lockedDisallowed) sb.append(LOCKED).append(",");
|
||||
if (this.networkDisallowed) sb.append(NETWORK).append(",");
|
||||
if (this.newsDisallowed) sb.append(NEWS).append(",");
|
||||
if (this.statusDisallowed) sb.append(STATUS).append(",");
|
||||
if (this.surftipsDisallowed) sb.append(SURFTIPS).append(",");
|
||||
if (this.wikiDisallowed) sb.append(WIKI).append(",");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public boolean isAllDisallowed() {
|
||||
return allDisallowed;
|
||||
}
|
||||
|
||||
public void setAllDisallowed(boolean allDisallowed) {
|
||||
this.allDisallowed = allDisallowed;
|
||||
}
|
||||
|
||||
public boolean isLockedDisallowed() {
|
||||
return lockedDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setLockedDisallowed(boolean lockedDisallowed) {
|
||||
this.lockedDisallowed = lockedDisallowed;
|
||||
}
|
||||
|
||||
public boolean isDirsDisallowed() {
|
||||
return dirsDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setDirsDisallowed(boolean dirsDisallowed) {
|
||||
this.dirsDisallowed = dirsDisallowed;
|
||||
}
|
||||
|
||||
public boolean isBlogDisallowed() {
|
||||
return blogDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setBlogDisallowed(boolean blogDisallowed) {
|
||||
this.blogDisallowed = blogDisallowed;
|
||||
}
|
||||
|
||||
public boolean isBookmarksDisallowed() {
|
||||
return bookmarksDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setBookmarksDisallowed(boolean bookmarksDisallowed) {
|
||||
this.bookmarksDisallowed = bookmarksDisallowed;
|
||||
}
|
||||
|
||||
public boolean isFileshareDisallowed() {
|
||||
return fileshareDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setFileshareDisallowed(boolean fileshareDisallowed) {
|
||||
this.fileshareDisallowed = fileshareDisallowed;
|
||||
}
|
||||
|
||||
public boolean isHomepageDisallowed() {
|
||||
return homepageDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setHomepageDisallowed(boolean homepageDisallowed) {
|
||||
this.homepageDisallowed = homepageDisallowed;
|
||||
}
|
||||
|
||||
public boolean isNetworkDisallowed() {
|
||||
return networkDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setNetworkDisallowed(boolean networkDisallowed) {
|
||||
this.networkDisallowed = networkDisallowed;
|
||||
}
|
||||
|
||||
public boolean isNewsDisallowed() {
|
||||
return newsDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setNewsDisallowed(boolean newsDisallowed) {
|
||||
this.newsDisallowed = newsDisallowed;
|
||||
}
|
||||
|
||||
public boolean isStatusDisallowed() {
|
||||
return statusDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setStatusDisallowed(boolean statusDisallowed) {
|
||||
this.statusDisallowed = statusDisallowed;
|
||||
}
|
||||
|
||||
public boolean isSurftipsDisallowed() {
|
||||
return surftipsDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setSurftipsDisallowed(boolean surftipsDisallowed) {
|
||||
this.surftipsDisallowed = surftipsDisallowed;
|
||||
}
|
||||
|
||||
public boolean isWikiDisallowed() {
|
||||
return wikiDisallowed || this.allDisallowed;
|
||||
}
|
||||
|
||||
public void setWikiDisallowed(boolean wikiDisallowed) {
|
||||
this.wikiDisallowed = wikiDisallowed;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue