boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) blacklist cleaner supports usage of regular expressions now

Browse Source 
*) refacored BlacklistCleaner_p.java for better readability
*) moved check of validity of patterns to the Balcklist implementation since patterns might be valid in one implementation, but not in another
*) added method to check validity to Blacklist interface
*) fixed some minor issues like typos or wrong whitespaces
*) set subversion properties for a whole bunch of files

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6359 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 16 years ago
parent 5a93807781
commit a6a3090c3d
25 changed files with 387 additions and 263 deletions
Show Stats Download Patch File Download Diff File

8
htroot/BlacklistCleaner_p.html
Unescape View File

@ -15,8 +15,12 @@
<fieldset class="selectList"><legend>Check list</legend>
<div style="display:inline;">
<select name="listNames" size="1">#{blacklists}#
<option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>#{/blacklists}#
<option value="#[name]#"#(selected)#:: selected="selected"#(/selected)#>#[name]#</option>#{/blacklists}#
</select>
<br /><br />
<input type="checkbox" name="allowRegex" #(checked)#:: checked="checked"#(/checked)# />
Allow regular expressions in host part of blacklist entries.
<br /><br />
<input type="submit" name="list" value="Check" />
</div>::
<p class="error">The blacklist-cleaner only works for the following blacklist-engines up to now:</p>
@ -40,11 +44,13 @@
<dt>
<label for="select#[entry]#" class="error">
#(error)#Two wildcards in host-part
::<!-- no error, should never be visible -->
::Either subdomain <u>or</u> wildcard
::Path is invalid Regex
::Wildcard not on begin or end
::Host contains illegal chars
::Double
::Host is invalid Regex
#(/error)#
</label>
<input type="checkbox" name="select#[entry]#" id="select#[entry]#" checked="checked" />

299
htroot/BlacklistCleaner_p.java
Unescape View File

@ -37,13 +37,11 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import de.anomic.data.AbstractBlacklist;
import de.anomic.data.Blacklist;
@ -54,6 +52,7 @@ import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.logging.Log;
import java.util.Set;
public class BlacklistCleaner_p {
@ -61,14 +60,7 @@ public class BlacklistCleaner_p {
private static final String DISABLED = "disabled_";
private static final String BLACKLISTS = "blacklists_";
private static final String ENTRIES = "entries_";
private static final int ERR_TWO_WILDCARDS_IN_HOST = 0;
private static final int ERR_SUBDOMAIN_XOR_WILDCARD = 1;
private static final int ERR_PATH_REGEX = 2;
private static final int ERR_WILDCARD_BEGIN_OR_END = 3;
private static final int ERR_HOST_WRONG_CHARS = 4;
private static final int ERR_DOUBLE_OCCURANCE = 5;
private final static String BLACKLIST_FILENAME_FILTER = "^.*\\.black$";
public static final Class<?>[] supportedBLEngines = {
@ -82,61 +74,75 @@ public class BlacklistCleaner_p {
listManager.switchboard = (Switchboard) env;
listManager.listsPath = new File(env.getRootPath(), env.getConfig("listManager.listsPath", "DATA/LISTS"));
String blacklistToUse = null;
// getting the list of supported blacklist types
final String supportedBlacklistTypesStr = AbstractBlacklist.BLACKLIST_TYPES_STRING;
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
if (post == null) {
prop.put("results", "0");
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
return prop;
}
if (post.containsKey("listNames")) {
blacklistToUse = post.get("listNames");
if (blacklistToUse.length() == 0 || !listManager.listSetContains("listManager.listsPath", blacklistToUse))
prop.put("results", "2");
}
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
if (blacklistToUse != null) {
prop.put("results", "1");
if (post.containsKey("delete")) {
prop.put(RESULTS + "modified", "1");
prop.put(RESULTS + "modified_delCount", removeEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select", true, true)));
} else if (post.containsKey("alter")) {
prop.put(RESULTS + "modified", "2");
prop.put(RESULTS + "modified_alterCount", alterEntries(blacklistToUse, supportedBlacklistTypes, getByPrefix(post, "select", true, false), getByPrefix(post, "entry", false, false)));
prop.put(DISABLED+"checked", "1");
if (post != null) {
final boolean allowRegex = post.get("allowRegex", "off").equalsIgnoreCase("on") ? true: false;
prop.put(DISABLED+"checked", (allowRegex) ? "1" : "0");
if (post.containsKey("listNames")) {
blacklistToUse = post.get("listNames");
if (blacklistToUse.length() == 0 || !listManager.listSetContains("listManager.listsPath", blacklistToUse)) {
prop.put("results", "2");
}
}
// list illegal entries
final HashMap<String, Integer> ies = getIllegalEntries(blacklistToUse, Switchboard.urlBlacklist);
prop.put(RESULTS + "blList", blacklistToUse);
prop.put(RESULTS + "entries", ies.size());
prop.putHTML(RESULTS + "blEngine", Switchboard.urlBlacklist.getEngineInfo());
prop.put(RESULTS + "disabled", (ies.size() == 0) ? "1" : "0");
if (ies.size() > 0) {
prop.put(RESULTS + DISABLED + "entries", ies.size());
int i = 0;
String s;
for (Entry<String, Integer> entry: ies.entrySet()) {
s = entry.getKey();
prop.put(RESULTS + DISABLED + ENTRIES + i + "_error", entry.getValue().longValue());
prop.putHTML(RESULTS + DISABLED + ENTRIES + i + "_entry", s);
i++;
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
if (blacklistToUse != null) {
prop.put("results", "1");
if (post.containsKey("delete")) {
prop.put(RESULTS + "modified", "1");
prop.put(RESULTS + "modified_delCount", removeEntries(blacklistToUse, supportedBlacklistTypes, getKeysByPrefix(post, "select", true)));
} else if (post.containsKey("alter")) {
prop.put(RESULTS + "modified", "2");
prop.put(RESULTS + "modified_alterCount", alterEntries(blacklistToUse, supportedBlacklistTypes, getKeysByPrefix(post, "select", false), getValuesByPrefix(post, "entry", false)));
}
// list illegal entries
final Map<String, Integer> illegalEntries = getIllegalEntries(blacklistToUse, Switchboard.urlBlacklist, allowRegex);
prop.put(RESULTS + "blList", blacklistToUse);
prop.put(RESULTS + "entries", illegalEntries.size());
prop.putHTML(RESULTS + "blEngine", Switchboard.urlBlacklist.getEngineInfo());
prop.put(RESULTS + "disabled", (illegalEntries.size() == 0) ? "1" : "0");
if (illegalEntries.size() > 0) {
prop.put(RESULTS + DISABLED + "entries", illegalEntries.size());
int i = 0;
String key;
for (Entry<String, Integer> entry : illegalEntries.entrySet()) {
key = entry.getKey();
prop.put(RESULTS + DISABLED + ENTRIES + i + "_error", entry.getValue().longValue());
prop.putHTML(RESULTS + DISABLED + ENTRIES + i + "_entry", key);
i++;
}
}
}
} else {
prop.put("results", "0");
putBlacklists(prop, listManager.getDirListing(listManager.listsPath, BLACKLIST_FILENAME_FILTER), blacklistToUse);
}
return prop;
}
/**
* Adds a list of blacklist to the server objects properties which are used to
* display the blacklist in the HTML page belonging to this servlet.
* @param prop Server objects properties object.
* @param lists List of blacklists.
* @param selected Element in list of blacklists which will be preselected in HTML.
*/
private static void putBlacklists(final serverObjects prop, final List<String> lists, final String selected) {
boolean supported = false;
for (int i=0; i<supportedBLEngines.length && !supported; i++) {
for (int i=0; i < supportedBLEngines.length && !supported; i++) {
supported |= (Switchboard.urlBlacklist.getClass() == supportedBLEngines[i]);
}
@ -155,13 +161,48 @@ public class BlacklistCleaner_p {
}
} else {
prop.put("disabled", "1");
for (int i=0; i<supportedBLEngines.length; i++) {
for (int i = 0; i < supportedBLEngines.length; i++) {
prop.putHTML(DISABLED + "engines_" + i + "_name", supportedBLEngines[i].getName());
}
prop.put(DISABLED + "engines", supportedBLEngines.length);
}
}
/**
* Retrieves all keys with a certain prefix from the data which has been sent and returns them as an array. This
* method is only a wrapper for {@link getByPrefix(de.anomic.server.serverObjects, java.lang.String, boolean, boolean)}
* which has been created to make it easier to understand the code.
* @param post All POST values.
* @param prefix Prefix by which the input is filtered.
* @param filterDoubles Set true if only unique results shall be returned, else false.
* @return Keys which have been posted.
*/
private static String[] getKeysByPrefix(final serverObjects post, final String prefix, final boolean filterDoubles) {
return getByPrefix(post, prefix, true, filterDoubles);
}
/**
* Retrieves all values with a certain prefix from the data which has been sent and returns them as an array. This
* method is only a wrapper for {@link getByPrefix(de.anomic.server.serverObjects, java.lang.String, boolean, boolean)}.
* @param post All POST values.
* @param prefix Prefix by which the input is filtered.
* @param filterDoubles Set true if only unique results shall be returned, else false.
* @return Values which have been posted.
*/
private static String[] getValuesByPrefix(final serverObjects post, final String prefix, final boolean filterDoubles) {
return getByPrefix(post, prefix, false, filterDoubles);
}
/**
* Method which does all the work for {@link getKeysByPrefix(de.anomic.server.serverObjects, java.lang.String prefix, boolean)}
* and {@link getValuesByPrefix(de.anomic.server.serverObjects, java.lang.String prefix, boolean)} which
* have been crested to make it easier to understand the code.
* @param post
* @param prefix
* @param useKeys
* @param useHashSet
* @return
*/
private static String[] getByPrefix(final serverObjects post, final String prefix, final boolean useKeys, final boolean useHashSet) {
Collection<String> r;
if (useHashSet) {
@ -169,20 +210,15 @@ public class BlacklistCleaner_p {
} else {
r = new ArrayList<String>();
}
String s;
if (useKeys) {
final Iterator<String> it = post.keySet().iterator();
while (it.hasNext()) {
if ((s = it.next()).indexOf(prefix) == 0) {
r.add(s.substring(prefix.length()));
for (String entry : post.keySet()) {
if (entry.indexOf(prefix) == 0) {
r.add(entry.substring(prefix.length()));
}
}
} else {
final Iterator<Map.Entry<String, String>> it = post.entrySet().iterator();
Map.Entry<String, String> entry;
while (it.hasNext()) {
entry = it.next();
for (Map.Entry<String, String> entry : post.entrySet()) {
if (entry.getKey().indexOf(prefix) == 0) {
r.add(entry.getValue());
}
@ -191,79 +227,56 @@ public class BlacklistCleaner_p {
return r.toArray(new String[r.size()]);
}
private static HashMap<String, Integer>/* entry, error-code */ getIllegalEntries(final String blacklistToUse, final Blacklist blEngine) {
final HashMap<String, Integer> r = new HashMap<String, Integer>();
final HashSet<String> ok = new HashSet<String>();
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
final Iterator<String> it = list.iterator();
String s, host, path;
/**
* Finds illegal entries in black list.
* @param blacklistToUse The blacklist to be checked.
* @param blEngine The blacklist engine which is used to check
* @param allowRegex Set to true to allow regular expressions in host part of blacklist entry.
* @return A map which contains all entries whoch have been identified as being
* illegal by the blacklistEngine with the entry as key and an error code as
* value.
*/
private static Map<String, Integer> getIllegalEntries(final String blacklistToUse, final Blacklist blEngine, final boolean allowRegex) {
final Map<String, Integer> illegalEntries = new HashMap<String, Integer>();
final Set<String> legalEntries = new HashSet<String>();
if (blEngine instanceof DefaultBlacklist) {
int slashPos;
while (it.hasNext()) {
s = (it.next()).trim();
// check for double-occurance
if (ok.contains(s)) {
r.put(s, Integer.valueOf(ERR_DOUBLE_OCCURANCE));
continue;
}
ok.add(s);
if ((slashPos = s.indexOf("/")) == -1) {
host = s;
path = ".*";
} else {
host = s.substring(0, slashPos);
path = s.substring(slashPos + 1);
}
final int i = host.indexOf("*");
final List<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
final Map<String, String> properties= new HashMap<String, String>();
properties.put("allowRegex", String.valueOf(allowRegex));
// check whether host begins illegally
if (!host.matches("([A-Za-z0-9_-]+|\\*)(\\.([A-Za-z0-9_-]+|\\*))*")) {
if (i == 0 && host.length() > 1 && host.charAt(1) != '.') {
r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD));
continue;
}
r.put(s, Integer.valueOf(ERR_HOST_WRONG_CHARS));
continue;
}
// in host-part only full sub-domains may be wildcards
if (host.length() > 0 && i > -1) {
if (!(i == 0 || i == host.length() - 1)) {
r.put(s, Integer.valueOf(ERR_WILDCARD_BEGIN_OR_END));
continue;
}
if (i == host.length() - 1 && host.length() > 1 && host.charAt(i - 1) != '.') {
r.put(s, Integer.valueOf(ERR_SUBDOMAIN_XOR_WILDCARD));
continue;
}
}
if (blEngine instanceof AbstractBlacklist) {
int err = 0;
for (String element : list) {
element = element.trim();
// check for double-occurences of "*" in host
if (host.indexOf("*", i + 1) > -1) {
r.put(s, Integer.valueOf(ERR_TWO_WILDCARDS_IN_HOST));
// check for double-occurance
if (legalEntries.contains(element)) {
illegalEntries.put(element, Integer.valueOf(AbstractBlacklist.ERR_DOUBLE_OCCURANCE));
continue;
}
// check for errors on regex-compiling path
try {
Pattern.compile(path);
} catch (final PatternSyntaxException e) {
r.put(s, Integer.valueOf(ERR_PATH_REGEX));
continue;
legalEntries.add(element);
err = blEngine.checkError(element, properties);
if (err > 0) {
illegalEntries.put(element, err);
}
}
}
return r;
return illegalEntries;
}
/**
* Removes existing entries from a blacklist.
* @param blacklistToUse The blacklist which contains the
* @param supportedBlacklistTypes Types of blacklists which the entry is to changed in.
* @param entries Array of entries to be deleted.
* @return Length of the list of entries to be removed.
*/
private static int removeEntries(final String blacklistToUse, final String[] supportedBlacklistTypes, final String[] entries) {
// load blacklist data from file
final ArrayList<String> list = listManager.getListArray(new File(listManager.listsPath, blacklistToUse));
@ -309,28 +322,36 @@ public class BlacklistCleaner_p {
}
return entries.length;
}
/**
* Changes existing entry in a blacklist.
* @param blacklistToUse The blacklist which contains the entry.
* @param supportedBlacklistTypes Types of blacklists which the entry is to changed in.
* @param oldEntry Entry to be changed.
* @param newEntry Changed entry.
* @return The length of the new entry.
*/
private static int alterEntries(
final String blacklistToUse,
final String[] supportedBlacklistTypes,
final String[] oldE,
final String[] newE) {
removeEntries(blacklistToUse, supportedBlacklistTypes, oldE);
final String[] oldEntry,
final String[] newEntry) {
removeEntries(blacklistToUse, supportedBlacklistTypes, oldEntry);
PrintWriter pw = null;
try {
pw = new PrintWriter(new FileWriter(new File(listManager.listsPath, blacklistToUse), true));
String host, path;
for (int i=0, pos; i<newE.length; i++) {
pos = newE[i].indexOf("/");
for (int i=0, pos; i<newEntry.length; i++) {
pos = newEntry[i].indexOf("/");
if (pos < 0) {
host = newE[i];
host = newEntry[i];
path = ".*";
} else {
host = newE[i].substring(0, pos);
path = newE[i].substring(pos + 1);
host = newEntry[i].substring(0, pos);
path = newEntry[i].substring(pos + 1);
}
pw.println(host + "/" + path);
for (int blTypes=0; blTypes < supportedBlacklistTypes.length; blTypes++) {
for (int blTypes = 0; blTypes < supportedBlacklistTypes.length; blTypes++) {
if (listManager.listSetContains(supportedBlacklistTypes[blTypes] + ".BlackLists",blacklistToUse)) {
Switchboard.urlBlacklist.add(
supportedBlacklistTypes[blTypes],
@ -343,6 +364,6 @@ public class BlacklistCleaner_p {
} catch (final IOException e) {
Log.logSevere("BLACKLIST-CLEANER", "error on writing altered entries to blacklist", e);
}
return newE.length;
return newEntry.length;
}
}

2
htroot/Blacklist_p.html
Unescape View File

@ -74,7 +74,7 @@
<input type="text" name="newEntry" size="50" />
<input type="submit" name="addBlacklistEntry" value="Add URL pattern" />
</div>
<p>The right '*', after the '/', can replaced by a <a href="http://java.sun.com/j2se/1.5.0/docs/api/java/util/regex/Pattern.html">regex</a>.</p>
<p>The right '*', after the '/', can be replaced by a <a href="http://java.sun.com/j2se/1.5.0/docs/api/java/util/regex/Pattern.html">regex</a>.</p>
<ul>
<li>domain.net/fullpath</li>
<li>domain.net/*</li>

4
htroot/Blacklist_p.java
Unescape View File

@ -43,7 +43,6 @@ import de.anomic.data.AbstractBlacklist;
import de.anomic.data.Blacklist;
import de.anomic.data.listManager;
import de.anomic.http.metadata.RequestHeader;
import de.anomic.search.SearchEventCache;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -65,9 +64,6 @@ public class Blacklist_p {
listManager.switchboard = (Switchboard) env;
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
// clean up all search events in case that a (new) blacklist entry denies previously returned results
SearchEventCache.cleanupEvents(true);
// getting the list of supported blacklist types
final String supportedBlacklistTypesStr = AbstractBlacklist.BLACKLIST_TYPES_STRING;
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");

6
htroot/IndexImportWikimedia_p.java
Unescape View File

@ -4,9 +4,9 @@
// first published 04.05.2009 on http://yacy.net
// Frankfurt, Germany
//
// $LastChangedDate: 2009-04-16 17:29:00 +0200 (Do, 16 Apr 2009) $
// $LastChangedRevision: 5812 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

6
htroot/IndexImport_p.java
Unescape View File

@ -7,9 +7,9 @@
//
//This file is contributed by Martin Thelian
//
// $LastChangedDate: 2005-10-17 17:46:12 +0200 (Mo, 17 Okt 2005) $
// $LastChangedRevision: 947 $
// $LastChangedBy: borg-0300 $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by

6
htroot/LogStatistics_p.java
Unescape View File

@ -8,9 +8,9 @@
//
// This File is contributed by Franz Brausze
//
// $LastChangedDate: 2007-01-17 12:00:00 +0100 (Di, 17 Jan 2007) $
// $LastChangedRevision: 3216 $
// $LastChangedBy: karlchenofhell $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

6
htroot/PerformanceGraph.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
htroot/PerformanceSearch_p.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

106
htroot/Ranking_p.java
Unescape View File

@ -5,9 +5,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2007-07-19 22:11:48 +0000 (Do, 19 Jul 2007) $
// $LastChangedRevision: 3995 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -44,38 +44,38 @@ public class Ranking_p {
private static final HashMap<String, String> rankingParameters = new HashMap<String, String>();
static {
rankingParameters.put(RankingProfile.APP_DC_CREATOR, "Appearance In Author");
rankingParameters.put(RankingProfile.APP_DC_TITLE, "Appearance In Title");
rankingParameters.put(RankingProfile.APPEMPH, "Appearance In Emphasized Text");
rankingParameters.put(RankingProfile.APP_DC_DESCRIPTION, "Appearance In Reference/Anchor Name");
rankingParameters.put(RankingProfile.APP_DC_SUBJECT, "Appearance In Tags");
rankingParameters.put(RankingProfile.APPURL, "Appearance In URL");
rankingParameters.put(RankingProfile.AUTHORITY, "Authority of Domain");
rankingParameters.put(RankingProfile.CATHASAPP, "Category App, Appearance");
rankingParameters.put(RankingProfile.CATHASAUDIO, "Category Audio Appearance");
rankingParameters.put(RankingProfile.CATHASIMAGE, "Category Image Appearance");
rankingParameters.put(RankingProfile.CATHASVIDEO, "Category Video Appearance");
rankingParameters.put(RankingProfile.CATINDEXOF, "Category Index Page");
rankingParameters.put(RankingProfile.DATE, "Date");
rankingParameters.put(RankingProfile.DESCRCOMPINTOPLIST, "Description Comp. Appears In Toplist");
rankingParameters.put(RankingProfile.DOMLENGTH, "Domain Length");
rankingParameters.put(RankingProfile.HITCOUNT, "Hit Count");
rankingParameters.put(RankingProfile.LLOCAL, "Links To Local Domain");
rankingParameters.put(RankingProfile.LOTHER, "Links To Other Domain");
rankingParameters.put(RankingProfile.PHRASESINTEXT, "Phrases In Text");
rankingParameters.put(RankingProfile.POSINTEXT, "Position In Text");
rankingParameters.put(RankingProfile.POSOFPHRASE, "Position Of Phrase");
rankingParameters.put(RankingProfile.POSINPHRASE, "Position In Phrase");
rankingParameters.put(RankingProfile.PREFER, "Application Of Prefer Pattern");
rankingParameters.put(RankingProfile.TERMFREQUENCY, "Term Frequency");
rankingParameters.put(RankingProfile.URLCOMPINTOPLIST, "URL Component Appears In Toplist");
rankingParameters.put(RankingProfile.URLCOMPS, "URL Components");
rankingParameters.put(RankingProfile.URLLENGTH, "URL Length");
rankingParameters.put(RankingProfile.WORDDISTANCE, "Word Distance");
rankingParameters.put(RankingProfile.WORDSINTEXT, "Words In Text");
rankingParameters.put(RankingProfile.WORDSINTITLE, "Words In Title");
rankingParameters.put(RankingProfile.YBR, "YaCy Block Rank");
rankingParameters.put(RankingProfile.LANGUAGE, "Preferred Language");
rankingParameters.put(RankingProfile.APP_DC_CREATOR, "Appearance In Author");
rankingParameters.put(RankingProfile.APP_DC_TITLE, "Appearance In Title");
rankingParameters.put(RankingProfile.APPEMPH, "Appearance In Emphasized Text");
rankingParameters.put(RankingProfile.APP_DC_DESCRIPTION, "Appearance In Reference/Anchor Name");
rankingParameters.put(RankingProfile.APP_DC_SUBJECT, "Appearance In Tags");
rankingParameters.put(RankingProfile.APPURL, "Appearance In URL");
rankingParameters.put(RankingProfile.AUTHORITY, "Authority of Domain");
rankingParameters.put(RankingProfile.CATHASAPP, "Category App, Appearance");
rankingParameters.put(RankingProfile.CATHASAUDIO, "Category Audio Appearance");
rankingParameters.put(RankingProfile.CATHASIMAGE, "Category Image Appearance");
rankingParameters.put(RankingProfile.CATHASVIDEO, "Category Video Appearance");
rankingParameters.put(RankingProfile.CATINDEXOF, "Category Index Page");
rankingParameters.put(RankingProfile.DATE, "Date");
rankingParameters.put(RankingProfile.DESCRCOMPINTOPLIST, "Description Comp. Appears In Toplist");
rankingParameters.put(RankingProfile.DOMLENGTH, "Domain Length");
rankingParameters.put(RankingProfile.HITCOUNT, "Hit Count");
rankingParameters.put(RankingProfile.LLOCAL, "Links To Local Domain");
rankingParameters.put(RankingProfile.LOTHER, "Links To Other Domain");
rankingParameters.put(RankingProfile.PHRASESINTEXT, "Phrases In Text");
rankingParameters.put(RankingProfile.POSINTEXT, "Position In Text");
rankingParameters.put(RankingProfile.POSOFPHRASE, "Position Of Phrase");
rankingParameters.put(RankingProfile.POSINPHRASE, "Position In Phrase");
rankingParameters.put(RankingProfile.PREFER, "Application Of Prefer Pattern");
rankingParameters.put(RankingProfile.TERMFREQUENCY, "Term Frequency");
rankingParameters.put(RankingProfile.URLCOMPINTOPLIST, "URL Component Appears In Toplist");
rankingParameters.put(RankingProfile.URLCOMPS, "URL Components");
rankingParameters.put(RankingProfile.URLLENGTH, "URL Length");
rankingParameters.put(RankingProfile.WORDDISTANCE, "Word Distance");
rankingParameters.put(RankingProfile.WORDSINTEXT, "Words In Text");
rankingParameters.put(RankingProfile.WORDSINTITLE, "Words In Title");
rankingParameters.put(RankingProfile.YBR, "YaCy Block Rank");
rankingParameters.put(RankingProfile.LANGUAGE, "Preferred Language");
}
private static serverObjects defaultValues() {
@ -108,24 +108,24 @@ public class Ranking_p {
String key;
int i, j = 0;
for (final Entry<String, String> entry: map.entrySet()) {
key = entry.getKey();
prop.put("attr" + attrExtension + "_" + j + "_name", rankingParameters.get(key.substring(prefix.length())));
prop.put("attr" + attrExtension + "_" + j + "_nameorg", key);
prop.put("attr" + attrExtension + "_" + j + "_select", maxRankingRange);
for (i=0; i<maxRankingRange; i++) {
prop.put("attr" + attrExtension + "_" + j + "_select_" + i + "_nameorg", key);
prop.put("attr" + attrExtension + "_" + j + "_select_" + i + "_value", i);
try {
prop.put("attr" + attrExtension + "_" + j + "_select_" + i + "_checked",
(i == Integer.valueOf(entry.getValue()).intValue()) ? "1" : "0");
} catch (final NumberFormatException e) {
prop.put("attr" + attrExtension + "_" + j + "_select_" + i + "_checked", "0");
}
}
prop.put("attr" + attrExtension + "_" + j + "_value",
Integer.valueOf(map.get(key)).intValue());
j++;
}
key = entry.getKey();
prop.put("attr" + attrExtension + "_" + j + "_name", rankingParameters.get(key.substring(prefix.length())));
prop.put("attr" + attrExtension + "_" + j + "_nameorg", key);
prop.put("attr" + attrExtension + "_" + j + "_select", maxRankingRange);
for (i=0; i<maxRankingRange; i++) {
prop.put("attr" + attrExtension + "_" + j + "_select_" + i + "_nameorg", key);
prop.put("attr" + attrExtension + "_" + j + "_select_" + i + "_value", i);
try {
prop.put("attr" + attrExtension + "_" + j + "_select_" + i + "_checked",
(i == Integer.valueOf(entry.getValue()).intValue()) ? "1" : "0");
} catch (final NumberFormatException e) {
prop.put("attr" + attrExtension + "_" + j + "_select_" + i + "_checked", "0");
}
}
prop.put("attr" + attrExtension + "_" + j + "_value",
Integer.valueOf(map.get(key)).intValue());
j++;
}
}
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {

6
htroot/SearchEventPicture.java
Unescape View File

@ -6,9 +6,9 @@
// Frankfurt, Germany, 2005
// Created 24.10.2005
//
// $LastChangedDate: 2005-10-23 19:50:27 +0200 (Sun, 23 Oct 2005) $
// $LastChangedRevision: 976 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

6
htroot/Supporter.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
htroot/Surftips.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
htroot/Threaddump_p.java
Unescape View File

@ -7,9 +7,9 @@
//
// This File is contributed by Alexander Fieger
//
// $LastChangedDate: 2008-01-22 12:51:43 +0100 (Di, 22 Jan 2008) $
// $LastChangedRevision: 4374 $
// $LastChangedBy: low012 $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

34
source/de/anomic/data/AbstractBlacklist.java
Unescape View File

@ -1,13 +1,13 @@
// indexAbstractReference.java
// AbstractBlacklist.java
// first published on http://www.yacy.net
// (C) 2007 by Bjoern Krombholz
// last major change: 12. August 2006 (theli) ?
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2009-01-30 23:44:20 +0100 (Fr, 30 Jan 2009) $
// $LastChangedRevision: 5543 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -41,10 +41,19 @@ import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import de.anomic.kelondro.util.SetTools;
import de.anomic.search.SearchEventCache;
import de.anomic.yacy.yacyURL;
public abstract class AbstractBlacklist implements Blacklist {
public static final int ERR_TWO_WILDCARDS_IN_HOST = 1;
public static final int ERR_SUBDOMAIN_XOR_WILDCARD = 2;
public static final int ERR_PATH_REGEX = 3;
public static final int ERR_WILDCARD_BEGIN_OR_END = 4;
public static final int ERR_HOST_WRONG_CHARS = 5;
public static final int ERR_DOUBLE_OCCURANCE = 6;
public static final int ERR_HOST_REGEX = 7;
protected static final HashSet<String> BLACKLIST_TYPES = new HashSet<String>(Arrays.asList(new String[]{
Blacklist.BLACKLIST_CRAWLER,
Blacklist.BLACKLIST_PROXY,
@ -117,6 +126,9 @@ public abstract class AbstractBlacklist implements Blacklist {
for(final Set<String> entry: this.cachedUrlHashs.values()) {
entry.clear();
}
// clean up all search events in case that an old blacklist entry denied previously returned results, but does not anymore
SearchEventCache.cleanupEvents(true);
}
public int size() {
@ -178,6 +190,8 @@ public abstract class AbstractBlacklist implements Blacklist {
}
}
}
// clean up all search events in case that a (new) blacklist entry denies previously returned results
SearchEventCache.cleanupEvents(true);
}
}
@ -191,6 +205,9 @@ public abstract class AbstractBlacklist implements Blacklist {
public void removeAll(final String blacklistType, final String host) {
getBlacklistMap(blacklistType,true).remove(host);
getBlacklistMap(blacklistType,false).remove(host);
// clean up all search events in case that an old blacklist entry denied previously returned results, but does not anymore
SearchEventCache.cleanupEvents(true);
}
public void remove(final String blacklistType, final String host, final String path) {
@ -209,7 +226,10 @@ public abstract class AbstractBlacklist implements Blacklist {
if (hostList.size() == 0)
blacklistMapNotMatch.remove(host);
}
}
// clean up all search events in case that an old blacklist entry denied previously returned results, but does not anymore
SearchEventCache.cleanupEvents(true);
}
public void add(final String blacklistType, String host, String path) {
if (host == null) throw new NullPointerException();
@ -227,6 +247,9 @@ public abstract class AbstractBlacklist implements Blacklist {
ArrayList<String> hostList = blacklistMap.get(host.toLowerCase());
if (hostList == null) blacklistMap.put(host.toLowerCase(), (hostList = new ArrayList<String>()));
hostList.add(path);
// clean up all search events in case that a (new) blacklist entry denies previously returned results
SearchEventCache.cleanupEvents(true);
}
public int blacklistCacheSize() {
@ -273,6 +296,7 @@ public abstract class AbstractBlacklist implements Blacklist {
}
return true;
}
public static boolean isMatchable (final String host) {
try {
if(Pattern.matches("^[a-z0-9.-]*$", host)) // simple Domain (yacy.net or www.yacy.net)

15
source/de/anomic/data/Blacklist.java
Unescape View File

@ -1,12 +1,12 @@
// indexReferenceBlacklist.java
// Blacklist.java
// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 26.03.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -29,6 +29,7 @@ package de.anomic.data;
import java.io.File;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import de.anomic.yacy.yacyURL;
@ -93,6 +94,8 @@ public interface Blacklist {
public boolean isListed(String blacklistType, yacyURL url);
public boolean isListed(String blacklistType, String hostlow, String path);
public boolean isListed(String blacklistType, String hostlow, String path);
public int checkError(String entry, Map<String, String> properties);
}

6
source/de/anomic/data/Coordinates.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

80
source/de/anomic/data/DefaultBlacklist.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-08-20 09:54:56 +0200 (Mi, 20 Aug 2008) $
// $LastChangedRevision: 5063 $
// $LastChangedBy: danielr $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
@ -29,6 +29,7 @@ package de.anomic.data;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@ -117,4 +118,77 @@ public class DefaultBlacklist extends AbstractBlacklist implements Blacklist {
}
return matched;
}
public int checkError(String element, Map<String, String> properties) {
boolean allowRegex = true;
int slashPos;
String host, path;
if (properties != null) {
allowRegex = properties.get("allowRegex").equalsIgnoreCase("true") ? true : false;
}
if ((slashPos = element.indexOf("/")) == -1) {
host = element;
path = ".*";
} else {
host = element.substring(0, slashPos);
path = element.substring(slashPos + 1);
}
if (!allowRegex || !isValidRegex(host)) {
final int i = host.indexOf("*");
// check whether host begins illegally
if (!host.matches("([A-Za-z0-9_-]+|\\*)(\\.([A-Za-z0-9_-]+|\\*))*")) {
if (i == 0 && host.length() > 1 && host.charAt(1) != '.') {
return ERR_SUBDOMAIN_XOR_WILDCARD;
}
return ERR_HOST_WRONG_CHARS;
}
// in host-part only full sub-domains may be wildcards
if (host.length() > 0 && i > -1) {
if (!(i == 0 || i == host.length() - 1)) {
return ERR_WILDCARD_BEGIN_OR_END;
}
if (i == host.length() - 1 && host.length() > 1 && host.charAt(i - 1) != '.') {
return ERR_SUBDOMAIN_XOR_WILDCARD;
}
}
// check for double-occurences of "*" in host
if (host.indexOf("*", i + 1) > -1) {
return ERR_TWO_WILDCARDS_IN_HOST;
}
} else if (allowRegex && !isValidRegex(host)) {
return ERR_HOST_REGEX;
}
// check for errors on regex-compiling path
if (!isValidRegex(path) && !path.equals("*")) {
return ERR_PATH_REGEX;
}
return 0;
}
/**
* Checks if a given expression is a valid regular expression.
* @param expression The expression to be checked.
* @return True if the expression is a valid regular expression, else false.
*/
private static boolean isValidRegex(String expression) {
boolean ret = true;
try {
Pattern.compile(expression);
} catch (final PatternSyntaxException e) {
ret = false;
}
return ret;
}
}

6
source/de/anomic/data/DidYouMeanLibrary.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
source/de/anomic/data/LibraryProvider.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
source/de/anomic/data/Location.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
source/de/anomic/data/OpenGeoDB.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
source/de/anomic/data/URLAnalysis.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2009-01-02 12:38:20 +0100 (Fr, 02 Jan 2009) $
// $LastChangedRevision: 5432 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
source/de/anomic/data/URLLicense.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
source/de/anomic/data/diff.java
Unescape View File

@ -8,9 +8,9 @@
//
// This file is contributed by Franz Brausze
//
// $LastChangedDate: $
// $LastChangedRevision: $
// $LastChangedBy: $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

Write Preview
Loading…
Cancel
Save