boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

big lists, accelerated 20x+;

Browse Source 
cleaned;

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@703 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 20 years ago
parent b7bccbff1d
commit 0de9245975
1 changed files with 192 additions and 200 deletions
Show Stats Download Patch File Download Diff File

220
htroot/Blacklist_p.java
Unescape View File

@ -6,7 +6,9 @@
// Frankfurt, Germany, 2004 // Frankfurt, Germany, 2004
// //
// This File is contributed by Alexander Schier // This File is contributed by Alexander Schier
// last change: 02.08.2004 //
// last major change: $LastChangedDate$ by $LastChangedBy$
// Revision: $LastChangedRevision$
// //
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by // it under the terms of the GNU General Public License as published by
@ -42,7 +44,7 @@
// Contributions and changes to the program code must be marked as such. // Contributions and changes to the program code must be marked as such.
// You must compile this file with // You must compile this file with
// javac -classpath .:../Classes Blacklist_p.java // javac -classpath .:../classes Blacklist_p.java
// if the shell's current path is HTROOT // if the shell's current path is HTROOT
import java.io.File; import java.io.File;
@ -50,10 +52,8 @@ import java.io.IOException;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.Iterator; import java.util.Iterator;
import java.util.Vector; import java.util.Vector;
import de.anomic.data.listManager; import de.anomic.data.listManager;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpdProxyHandler;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore; import de.anomic.server.serverCore;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -62,134 +62,124 @@ import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeed;
public class Blacklist_p { public class Blacklist_p {
private final static String BLACKLIST = "blackLists_";
private final static String BLACKLIST_ALL = "proxyBlackLists";
private final static String BLACKLIST_ACTIVE = "proxyBlackListsActive";
private final static String BLACKLIST_SHARED = "proxyBlackListsShared";
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements // return variable that accumulates replacements
listManager.switchboard = (plasmaSwitchboard) env; listManager.switchboard = (plasmaSwitchboard) env;
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS")); listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
final serverObjects prop = new serverObjects();
serverObjects prop = new serverObjects();
String line; String line;
String HTMLout = ""; // String HTMLout = "";
String out = "";
String removeItem = "removeme"; String removeItem = "removeme";
int numItems=0; int numItems = 0;
int i=0; int i; // need below
String filenames[] = listManager.getListslistArray("proxyBlackLists"); String[] filenames = listManager.getListslistArray(BLACKLIST_ALL);
String filename = ""; String filename = "";
if (post != null) {
if(post != null && post.containsKey("blackLists")){ //Blacklist selected if (post.containsKey("blackLists")) { // Blacklist selected
filename = (String)post.get("blackLists"); filename = (String)post.get("blackLists");
}else if(post != null && post.containsKey("filename")){ } else if (post.containsKey("filename")) {
filename = (String)post.get("filename"); filename = (String)post.get("filename");
}else if(filenames.length > 0){ //first BlackList } else if (filenames.length > 0){ // first BlackList
filename = filenames[0]; filename = filenames[0];
}else{ //No BlackList // } else { //No BlackList
//No file // System.out.println("DEBUG: No Blacklist found");
filename = ""; //?
System.out.println("DEBUG: No Blacklist found");
} }
prop.put("status", 0);//nothing prop.put("status", 0); // nothing
// del list
//List Management if (post.containsKey("dellistbutton")) {
final File BlackListFile = new File(listManager.listsPath, filename);
//Del list
if( post != null && post.containsKey("dellistbutton") ){
File BlackListFile = new File(listManager.listsPath, filename);
BlackListFile.delete(); BlackListFile.delete();
//Remove from all BlackLists Lists // remove from all BlackLists Lists
listManager.removeListFromListslist("proxyBlackLists", filename); listManager.removeListFromListslist(BLACKLIST_ALL, filename);
listManager.removeListFromListslist("proxyBlackListsActive", filename); listManager.removeListFromListslist(BLACKLIST_ACTIVE, filename);
listManager.removeListFromListslist("proxyBlackListsShared", filename); listManager.removeListFromListslist(BLACKLIST_SHARED, filename);
//reload Blacklists // reload Blacklists
listManager.reloadBlacklists(); listManager.reloadBlacklists();
filenames = listManager.getListslistArray(BLACKLIST_ALL);
filenames = listManager.getListslistArray("proxyBlackLists"); if (filenames.length > 0) {
if(filenames.length > 0){
filename = filenames[0]; filename = filenames[0];
} }
}//del list
if( post != null && post.containsKey("newlistbutton") ){
// new list
} else if (post.containsKey("newlistbutton")) {
String newList = (String)post.get("newlist"); String newList = (String)post.get("newlist");
if( !newList.endsWith(".black") ){ if (!newList.endsWith(".black")) {
newList += ".black"; newList += ".black";
} }
filename = newList; //to select it in the returnes Document filename = newList; //to select it in the returnes Document
try{ try {
File newFile = new File(listManager.listsPath, newList); final File newFile = new File(listManager.listsPath, newList);
newFile.createNewFile(); newFile.createNewFile();
listManager.addListToListslist(BLACKLIST_ALL, newList);
listManager.addListToListslist(BLACKLIST_ACTIVE, newList);
listManager.addListToListslist(BLACKLIST_SHARED, newList);
} catch (IOException e) {}
listManager.addListToListslist("proxyBlackLists", newList);
listManager.addListToListslist("proxyBlackListsActive", newList);
listManager.addListToListslist("proxyBlackListsShared", newList);
}catch(IOException e){} } else if (post.containsKey("activatelistbutton")) {
if( listManager.ListInListslist(BLACKLIST_ACTIVE, filename) ){
}//newlist listManager.removeListFromListslist(BLACKLIST_ACTIVE, filename);
} else { // inactive list -> enable
if( post != null && post.containsKey("activatelistbutton") ){ listManager.addListToListslist(BLACKLIST_ACTIVE, filename);
if( listManager.ListInListslist("proxyBlackListsActive", filename) ){
listManager.removeListFromListslist("proxyBlackListsActive", filename);
}else{ //inactive list -> enable
listManager.addListToListslist("proxyBlackListsActive", filename);
} }
listManager.reloadBlacklists(); listManager.reloadBlacklists();
}
if( post != null && post.containsKey("sharelistbutton") ){
if( listManager.ListInListslist("proxyBlackListsShared", filename) ){ } else if (post.containsKey("sharelistbutton")) {
//Remove from shared BlackLists if (listManager.ListInListslist(BLACKLIST_SHARED, filename)) {
listManager.removeListFromListslist("proxyBlackListsShared", filename); // Remove from shared BlackLists
}else{ //inactive list -> enable listManager.removeListFromListslist(BLACKLIST_SHARED, filename);
listManager.addListToListslist("proxyBlackListsShared", filename); } else { // inactive list -> enable
listManager.addListToListslist(BLACKLIST_SHARED, filename);
} }
} } // List Management End
//List Management End
Vector list = listManager.getListArray(new File(listManager.listsPath, filename)); // remove a Item?
//remove a Item? if (post.containsKey("delbutton") &&
if( post != null && post.containsKey("delbutton") && post.containsKey("Itemlist") && !((String)post.get("Itemlist")).equals("") ){ post.containsKey("Itemlist") &&
!((String)post.get("Itemlist")).equals("") ) {
removeItem = (String)post.get("Itemlist"); removeItem = (String)post.get("Itemlist");
} }
} // post != null
//Read the List // Read the List
Iterator it = list.iterator(); final Vector list = listManager.getListArray(new File(listManager.listsPath, filename));
while(it.hasNext()){ final StringBuffer out = new StringBuffer(list.size() * 128);
line = (String) it.next(); final Iterator iter = list.iterator();
while (iter.hasNext()){
line = (String) iter.next();
if(! (line.startsWith("#") || line.equals("") || line.equals(removeItem)) ){ //Not the item to remove if (!(line.length() == 0 || line.charAt(0) == '#' || line.equals(removeItem))) { //Not the item to remove
prop.put("Itemlist_"+numItems+"_item", line); prop.put("Itemlist_" + numItems + "_item", line);
numItems++; numItems++;
} }
if(! line.equals(removeItem) ){ if (line.equals(removeItem)) {
out += line + serverCore.crlfString; //full list
}else{
prop.put("status", 1);//removed prop.put("status", 1);//removed
prop.put("status_item", line); prop.put("status_item", line);
if (listManager.switchboard.urlBlacklist != null) // if (listManager.switchboard.urlBlacklist != null) {
listManager.switchboard.urlBlacklist.remove(line); // listManager.switchboard.urlBlacklist.remove(line);
if (plasmaSwitchboard.urlBlacklist != null) {
plasmaSwitchboard.urlBlacklist.remove(line);
}
} else {
out.append(line).append(serverCore.crlfString); //full list
} }
} }
prop.put("Itemlist", numItems); prop.put("Itemlist", numItems);
//Add a new Item // Add a new Item
if( post != null && post.containsKey("addbutton") && !((String)post.get("newItem")).equals("") ){ if (post != null && post.containsKey("addbutton") && !((String)post.get("newItem")).equals("")) {
String newItem = (String)post.get("newItem"); String newItem = (String)post.get("newItem");
//clean http:// //clean http://
@ -205,7 +195,7 @@ public class Blacklist_p {
newItem = newItem + "/.*"; newItem = newItem + "/.*";
} }
out += newItem+"\n"; out.append(newItem).append(serverCore.crlfString);
prop.put("Itemlist_"+numItems+"_item", newItem); prop.put("Itemlist_"+numItems+"_item", newItem);
numItems++; numItems++;
@ -214,51 +204,53 @@ public class Blacklist_p {
prop.put("status", 2);//added prop.put("status", 2);//added
prop.put("status_item", newItem);//added prop.put("status_item", newItem);//added
//add to blacklist // add to blacklist
if (listManager.switchboard.urlBlacklist != null) // if (listManager.switchboard.urlBlacklist != null)
listManager.switchboard.urlBlacklist.add(newItem.substring(0, pos), newItem.substring(pos + 1)); // listManager.switchboard.urlBlacklist.add(newItem.substring(0, pos), newItem.substring(pos + 1));
if (plasmaSwitchboard.urlBlacklist != null) {
plasmaSwitchboard.urlBlacklist.add(newItem.substring(0, pos), newItem.substring(pos + 1));
}
} }
listManager.writeList(new File(listManager.listsPath, filename), out); listManager.writeList(new File(listManager.listsPath, filename), out.toString());
//List known hosts for BlackList retrieval // List known hosts for BlackList retrieval
yacySeed seed; yacySeed seed;
if( yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0 ){ //no nullpointer error if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { // no nullpointer error
Enumeration e = yacyCore.seedDB.seedsConnected(true, false, null); final Enumeration e = yacyCore.seedDB.seedsConnected(true, false, null);
i=0; i = 0;
while (e.hasMoreElements()) { while (e.hasMoreElements()) {
seed = (yacySeed) e.nextElement(); seed = (yacySeed) e.nextElement();
if (seed != null) { if (seed != null) {
String Hash = seed.hash; final String Hash = seed.hash;
String Name = seed.get("Name", "nameless"); final String Name = seed.get("Name", "nameless");
prop.put("otherHosts_"+i+"_hash", Hash); prop.put("otherHosts_" + i + "_hash", Hash);
prop.put("otherHosts_"+i+"_name", Name); prop.put("otherHosts_" + i + "_name", Name);
i++; i++;
} }
} }
prop.put("otherHosts", i); prop.put("otherHosts", i);
}else{ // } else {
//DEBUG: System.out.println("BlackList_p: yacy seed not loaded!"); // System.out.println("BlackList_p: yacy seed not loaded!"); // DEBUG:
} }
String BlackLists[] = listManager.getListslistArray("proxyBlackLists");
//List BlackLists // List BlackLists
for(i=0; i <= BlackLists.length -1;i++){ final String[] BlackLists = listManager.getListslistArray(BLACKLIST_ALL);
prop.put("blackLists_"+i+"_name", BlackLists[i]); for (i = 0; i <= BlackLists.length - 1; i++) {
prop.put("blackLists_"+i+"_active", 0); prop.put(BLACKLIST + i + "_name", BlackLists[i]);
prop.put("blackLists_"+i+"_shared", 0); prop.put(BLACKLIST + i + "_active", 0);
prop.put("blackLists_"+i+"_selected", 0); prop.put(BLACKLIST + i + "_shared", 0);
if( BlackLists[i].equals(filename) ){ //current List prop.put(BLACKLIST + i + "_selected", 0);
prop.put("blackLists_"+i+"_selected", 1); if (BlackLists[i].equals(filename)) { //current List
prop.put(BLACKLIST + i + "_selected", 1);
} }
if( listManager.ListInListslist("proxyBlackListsActive", BlackLists[i]) ){ if (listManager.ListInListslist(BLACKLIST_ACTIVE, BlackLists[i])) {
prop.put("blackLists_"+i+"_active", 1); prop.put(BLACKLIST + i + "_active", 1);
} }
if( listManager.ListInListslist("proxyBlackListsShared", BlackLists[i]) ){ if (listManager.ListInListslist(BLACKLIST_SHARED, BlackLists[i])) {
prop.put("blackLists_"+i+"_shared", 1); prop.put(BLACKLIST + i + "_shared", 1);
} }
} }
prop.put("blackLists", i); prop.put("blackLists", i);
prop.put("filename", filename); prop.put("filename", filename);
return prop; return prop;
} }

Write Preview
Loading…
Cancel
Save