boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

big lists, accelerated 20x+;

Browse Source 
cleaned;

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@703 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 20 years ago
parent b7bccbff1d
commit 0de9245975
1 changed files with 192 additions and 200 deletions
Show Stats Download Patch File Download Diff File

392
htroot/Blacklist_p.java
Unescape View File

@ -6,7 +6,9 @@
// Frankfurt, Germany, 2004 // Frankfurt, Germany, 2004
// //
// This File is contributed by Alexander Schier // This File is contributed by Alexander Schier
// last change: 02.08.2004 //
// last major change: $LastChangedDate$ by $LastChangedBy$
// Revision: $LastChangedRevision$
// //
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by // it under the terms of the GNU General Public License as published by
@ -42,7 +44,7 @@
// Contributions and changes to the program code must be marked as such. // Contributions and changes to the program code must be marked as such.
// You must compile this file with // You must compile this file with
// javac -classpath .:../Classes Blacklist_p.java // javac -classpath .:../classes Blacklist_p.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.io.File; import java.io.File;
@ -50,10 +52,8 @@ import java.io.IOException;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.Iterator; import java.util.Iterator;
import java.util.Vector; import java.util.Vector;
import de.anomic.data.listManager; import de.anomic.data.listManager;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpdProxyHandler;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -62,205 +62,197 @@ import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeed;
public class Blacklist_p { public class Blacklist_p {
private final static String BLACKLIST = "blackLists_";
private final static String BLACKLIST_ALL = "proxyBlackLists";
private final static String BLACKLIST_ACTIVE = "proxyBlackListsActive";
private final static String BLACKLIST_SHARED = "proxyBlackListsShared";
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
listManager.switchboard = (plasmaSwitchboard) env;
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
final serverObjects prop = new serverObjects();
String line;
// String HTMLout = "";
String removeItem = "removeme";
int numItems = 0;
int i; // need below
String[] filenames = listManager.getListslistArray(BLACKLIST_ALL);
String filename = "";
if (post != null) {
if (post.containsKey("blackLists")) { // Blacklist selected
filename = (String)post.get("blackLists");
} else if (post.containsKey("filename")) {
filename = (String)post.get("filename");
} else if (filenames.length > 0){ // first BlackList
filename = filenames[0];
// } else { //No BlackList
// System.out.println("DEBUG: No Blacklist found");
}
prop.put("status", 0); // nothing
// del list
if (post.containsKey("dellistbutton")) {
final File BlackListFile = new File(listManager.listsPath, filename);
BlackListFile.delete();
// remove from all BlackLists Lists
listManager.removeListFromListslist(BLACKLIST_ALL, filename);
listManager.removeListFromListslist(BLACKLIST_ACTIVE, filename);
listManager.removeListFromListslist(BLACKLIST_SHARED, filename);
// reload Blacklists
listManager.reloadBlacklists();
filenames = listManager.getListslistArray(BLACKLIST_ALL);
if (filenames.length > 0) {
filename = filenames[0];
}
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { // new list
// return variable that accumulates replacements } else if (post.containsKey("newlistbutton")) {
listManager.switchboard = (plasmaSwitchboard) env; String newList = (String)post.get("newlist");
if (!newList.endsWith(".black")) {
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS")); newList += ".black";
}
serverObjects prop = new serverObjects(); filename = newList; //to select it in the returnes Document
String line; try {
String HTMLout = ""; final File newFile = new File(listManager.listsPath, newList);
String out = ""; newFile.createNewFile();
String removeItem = "removeme"; listManager.addListToListslist(BLACKLIST_ALL, newList);
int numItems=0; listManager.addListToListslist(BLACKLIST_ACTIVE, newList);
int i=0; listManager.addListToListslist(BLACKLIST_SHARED, newList);
} catch (IOException e) {}
String filenames[] = listManager.getListslistArray("proxyBlackLists");
String filename = "";
} else if (post.containsKey("activatelistbutton")) {
if( listManager.ListInListslist(BLACKLIST_ACTIVE, filename) ){
if(post != null && post.containsKey("blackLists")){ //Blacklist selected listManager.removeListFromListslist(BLACKLIST_ACTIVE, filename);
filename = (String)post.get("blackLists"); } else { // inactive list -> enable
}else if(post != null && post.containsKey("filename")){ listManager.addListToListslist(BLACKLIST_ACTIVE, filename);
filename = (String)post.get("filename"); }
}else if(filenames.length > 0){ //first BlackList listManager.reloadBlacklists();
filename = filenames[0];
}else{ //No BlackList } else if (post.containsKey("sharelistbutton")) {
//No file if (listManager.ListInListslist(BLACKLIST_SHARED, filename)) {
filename = ""; //? // Remove from shared BlackLists
System.out.println("DEBUG: No Blacklist found"); listManager.removeListFromListslist(BLACKLIST_SHARED, filename);
} } else { // inactive list -> enable
prop.put("status", 0);//nothing listManager.addListToListslist(BLACKLIST_SHARED, filename);
}
} // List Management End
//List Management
// remove a Item?
//Del list if (post.containsKey("delbutton") &&
if( post != null && post.containsKey("dellistbutton") ){ post.containsKey("Itemlist") &&
!((String)post.get("Itemlist")).equals("") ) {
File BlackListFile = new File(listManager.listsPath, filename); removeItem = (String)post.get("Itemlist");
BlackListFile.delete(); }
} // post != null
//Remove from all BlackLists Lists
listManager.removeListFromListslist("proxyBlackLists", filename); // Read the List
listManager.removeListFromListslist("proxyBlackListsActive", filename); final Vector list = listManager.getListArray(new File(listManager.listsPath, filename));
listManager.removeListFromListslist("proxyBlackListsShared", filename); final StringBuffer out = new StringBuffer(list.size() * 128);
final Iterator iter = list.iterator();
//reload Blacklists while (iter.hasNext()){
listManager.reloadBlacklists(); line = (String) iter.next();
filenames = listManager.getListslistArray("proxyBlackLists"); if (!(line.length() == 0 || line.charAt(0) == '#' || line.equals(removeItem))) { //Not the item to remove
if(filenames.length > 0){ prop.put("Itemlist_" + numItems + "_item", line);
filename = filenames[0]; numItems++;
} }
}//del list
if (line.equals(removeItem)) {
if( post != null && post.containsKey("newlistbutton") ){ prop.put("status", 1);//removed
prop.put("status_item", line);
String newList = (String)post.get("newlist"); // if (listManager.switchboard.urlBlacklist != null) {
if( !newList.endsWith(".black") ){ // listManager.switchboard.urlBlacklist.remove(line);
newList += ".black"; if (plasmaSwitchboard.urlBlacklist != null) {
} plasmaSwitchboard.urlBlacklist.remove(line);
}
filename = newList; //to select it in the returnes Document } else {
try{ out.append(line).append(serverCore.crlfString); //full list
File newFile = new File(listManager.listsPath, newList); }
newFile.createNewFile(); }
prop.put("Itemlist", numItems);
listManager.addListToListslist("proxyBlackLists", newList);
listManager.addListToListslist("proxyBlackListsActive", newList); // Add a new Item
listManager.addListToListslist("proxyBlackListsShared", newList); if (post != null && post.containsKey("addbutton") && !((String)post.get("newItem")).equals("")) {
String newItem = (String)post.get("newItem");
}catch(IOException e){}
//clean http://
}//newlist if ( newItem.startsWith("http://") ){
newItem = newItem.substring(7);
if( post != null && post.containsKey("activatelistbutton") ){ }
if( listManager.ListInListslist("proxyBlackListsActive", filename) ){ //append "/.*"
listManager.removeListFromListslist("proxyBlackListsActive", filename); int pos = newItem.indexOf("/");
}else{ //inactive list -> enable if (pos < 0) {
listManager.addListToListslist("proxyBlackListsActive", filename); // add default empty path pattern
} pos = newItem.length();
newItem = newItem + "/.*";
listManager.reloadBlacklists(); }
}
out.append(newItem).append(serverCore.crlfString);
if( post != null && post.containsKey("sharelistbutton") ){
prop.put("Itemlist_"+numItems+"_item", newItem);
if( listManager.ListInListslist("proxyBlackListsShared", filename) ){ numItems++;
//Remove from shared BlackLists prop.put("Itemlist", numItems);
listManager.removeListFromListslist("proxyBlackListsShared", filename);
}else{ //inactive list -> enable prop.put("status", 2);//added
listManager.addListToListslist("proxyBlackListsShared", filename); prop.put("status_item", newItem);//added
}
} // add to blacklist
//List Management End // if (listManager.switchboard.urlBlacklist != null)
// listManager.switchboard.urlBlacklist.add(newItem.substring(0, pos), newItem.substring(pos + 1));
if (plasmaSwitchboard.urlBlacklist != null) {
plasmaSwitchboard.urlBlacklist.add(newItem.substring(0, pos), newItem.substring(pos + 1));
Vector list = listManager.getListArray(new File(listManager.listsPath, filename)); }
//remove a Item? }
if( post != null && post.containsKey("delbutton") && post.containsKey("Itemlist") && !((String)post.get("Itemlist")).equals("") ){ listManager.writeList(new File(listManager.listsPath, filename), out.toString());
removeItem = (String)post.get("Itemlist");
} // List known hosts for BlackList retrieval
yacySeed seed;
//Read the List if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { // no nullpointer error
Iterator it = list.iterator(); final Enumeration e = yacyCore.seedDB.seedsConnected(true, false, null);
while(it.hasNext()){ i = 0;
line = (String) it.next(); while (e.hasMoreElements()) {
seed = (yacySeed) e.nextElement();
if(! (line.startsWith("#") || line.equals("") || line.equals(removeItem)) ){ //Not the item to remove
prop.put("Itemlist_"+numItems+"_item", line);
numItems++;
}
if(! line.equals(removeItem) ){
out += line + serverCore.crlfString; //full list
}else{
prop.put("status", 1);//removed
prop.put("status_item", line);
if (listManager.switchboard.urlBlacklist != null)
listManager.switchboard.urlBlacklist.remove(line);
}
}
prop.put("Itemlist", numItems);
//Add a new Item
if( post != null && post.containsKey("addbutton") && !((String)post.get("newItem")).equals("") ){
String newItem = (String)post.get("newItem");
//clean http://
if ( newItem.startsWith("http://") ){
newItem = newItem.substring(7);
}
//append "/.*"
int pos = newItem.indexOf("/");
if (pos < 0) {
// add default empty path pattern
pos = newItem.length();
newItem = newItem + "/.*";
}
out += newItem+"\n";
prop.put("Itemlist_"+numItems+"_item", newItem);
numItems++;
prop.put("Itemlist", numItems);
prop.put("status", 2);//added
prop.put("status_item", newItem);//added
//add to blacklist
if (listManager.switchboard.urlBlacklist != null)
listManager.switchboard.urlBlacklist.add(newItem.substring(0, pos), newItem.substring(pos + 1));
}
listManager.writeList(new File(listManager.listsPath, filename), out);
//List known hosts for BlackList retrieval
yacySeed seed;
if( yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0 ){ //no nullpointer error
Enumeration e = yacyCore.seedDB.seedsConnected(true, false, null);
i=0;
while (e.hasMoreElements()) {
seed = (yacySeed) e.nextElement();
if (seed != null) { if (seed != null) {
String Hash = seed.hash; final String Hash = seed.hash;
String Name = seed.get("Name", "nameless"); final String Name = seed.get("Name", "nameless");
prop.put("otherHosts_"+i+"_hash", Hash); prop.put("otherHosts_" + i + "_hash", Hash);
prop.put("otherHosts_"+i+"_name", Name); prop.put("otherHosts_" + i + "_name", Name);
i++; i++;
} }
} }
prop.put("otherHosts", i); prop.put("otherHosts", i);
}else{ // } else {
//DEBUG: System.out.println("BlackList_p: yacy seed not loaded!"); // System.out.println("BlackList_p: yacy seed not loaded!"); // DEBUG:
} }
String BlackLists[] = listManager.getListslistArray("proxyBlackLists");
// List BlackLists
//List BlackLists final String[] BlackLists = listManager.getListslistArray(BLACKLIST_ALL);
for(i=0; i <= BlackLists.length -1;i++){ for (i = 0; i <= BlackLists.length - 1; i++) {
prop.put("blackLists_"+i+"_name", BlackLists[i]); prop.put(BLACKLIST + i + "_name", BlackLists[i]);
prop.put("blackLists_"+i+"_active", 0); prop.put(BLACKLIST + i + "_active", 0);
prop.put("blackLists_"+i+"_shared", 0); prop.put(BLACKLIST + i + "_shared", 0);
prop.put("blackLists_"+i+"_selected", 0); prop.put(BLACKLIST + i + "_selected", 0);
if( BlackLists[i].equals(filename) ){ //current List if (BlackLists[i].equals(filename)) { //current List
prop.put("blackLists_"+i+"_selected", 1); prop.put(BLACKLIST + i + "_selected", 1);
} }
if( listManager.ListInListslist("proxyBlackListsActive", BlackLists[i]) ){ if (listManager.ListInListslist(BLACKLIST_ACTIVE, BlackLists[i])) {
prop.put("blackLists_"+i+"_active", 1); prop.put(BLACKLIST + i + "_active", 1);
} }
if( listManager.ListInListslist("proxyBlackListsShared", BlackLists[i]) ){ if (listManager.ListInListslist(BLACKLIST_SHARED, BlackLists[i])) {
prop.put("blackLists_"+i+"_shared", 1); prop.put(BLACKLIST + i + "_shared", 1);
} }
} }
prop.put("blackLists", i); prop.put("blackLists", i);
prop.put("filename", filename);
prop.put("filename", filename); return prop;
return prop;
} }
} }

Write Preview
Loading…
Cancel
Save