boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) added new user right: extended search right (allows to define users who can query more results than anonymous users)

Browse Source 
*) cleaned up code a little bit

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7635 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 14 years ago
parent 564184909a
commit 1ff9947f91
16 changed files with 515 additions and 308 deletions
Show Stats Download Patch File Download Diff File

2
htroot/Blog.java
Unescape View File

@ -92,7 +92,7 @@ public class Blog {
if (!hasRights) {
final UserDB.Entry userentry = sb.userDB.proxyAuth(header.get(RequestHeader.AUTHORIZATION, "xxxxxx"));
if (userentry != null && userentry.hasRight(UserDB.Entry.BLOG_RIGHT)) {
if (userentry != null && userentry.hasRight(UserDB.AccessRight.BLOG_RIGHT)) {
hasRights=true;
} else if (post.containsKey("login")) {
//opens login window if login link is clicked

2
htroot/BlogComments.java
Unescape View File

@ -76,7 +76,7 @@ public class BlogComments {
if (!hasRights) {
final UserDB.Entry userentry = sb.userDB.proxyAuth(header.get(RequestHeader.AUTHORIZATION, "xxxxxx"));
if (userentry != null && userentry.hasRight(UserDB.Entry.BLOG_RIGHT)) {
if (userentry != null && userentry.hasRight(UserDB.AccessRight.BLOG_RIGHT)) {
hasRights = true;
}
//opens login window if login link is clicked

2
htroot/Bookmarks.java
Unescape View File

@ -85,7 +85,7 @@ public class Bookmarks {
prop.clear();
sb = (Switchboard) env;
user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user!= null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT));
isAdmin = (sb.verifyAuthentication(header, true) || user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name
if (user != null) {

28
htroot/ConfigAccounts_p.java
Unescape View File

@ -38,10 +38,12 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Digest;
import de.anomic.data.UserDB;
import de.anomic.data.UserDB.AccessRight;
import de.anomic.http.server.HTTPDemon;
import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import java.util.EnumMap;
import java.util.Map;
public class ConfigAccounts_p {
@ -103,12 +105,12 @@ public class ConfigAccounts_p {
prop.put("address", "");
prop.put("timelimit", "");
prop.put("timeused", "");
final String[] rightNames = UserDB.Entry.RIGHT_NAMES.split(",");
final String[] rights = UserDB.Entry.RIGHT_TYPES.split(",");
final AccessRight[] rights = AccessRight.values();
int c = 0;
for (final String right : rights) {
prop.put("rights_" + c + "_name", right);
prop.put("rights_" + c +"_friendlyName", rightNames[c]);
for (final AccessRight right : rights) {
prop.put("rights_" + c + "_name", right.toString());
prop.put("rights_" + c +"_friendlyName", right.getFriendlyName());
prop.put("rights_" + c + "_set", "0");
c++;
}
@ -142,7 +144,7 @@ public class ConfigAccounts_p {
prop.put("timelimit", entry.getTimeLimit());
prop.put("timeused", entry.getTimeUsed());
int count = 0;
for (final String right : rights){
for (final AccessRight right : rights){
prop.put("rights_" + count + "_set", entry.hasRight(right) ? "1" : "0");
count++;
}
@ -169,10 +171,10 @@ public class ConfigAccounts_p {
final String address = post.get("address");
final String timeLimit = post.get("timelimit");
final String timeUsed = post.get("timeused");
final Map<String, String> rightsSet = new HashMap<String, String>();
final Map<AccessRight, String> rightsSet = new EnumMap<AccessRight, String>(AccessRight.class);
for(final String right : rights) {
rightsSet.put(right, post.containsKey(right) && "on".equals(post.get(right)) ? "true" : "false");
for(final AccessRight right : rights) {
rightsSet.put(right, post.containsKey(right.toString()) && "on".equals(post.get(right.toString())) ? "true" : "false");
}
final Map<String, String> mem = new HashMap<String, String>();
@ -188,8 +190,8 @@ public class ConfigAccounts_p {
mem.put(UserDB.Entry.TIME_LIMIT, timeLimit);
mem.put(UserDB.Entry.TIME_USED, timeUsed);
for (final String right : rights) {
mem.put(right, rightsSet.get(right));
for (final AccessRight right : rights) {
mem.put(right.toString(), rightsSet.get(right));
}
try {
@ -217,8 +219,8 @@ public class ConfigAccounts_p {
entry.setProperty(UserDB.Entry.TIME_LIMIT, timeLimit);
entry.setProperty(UserDB.Entry.TIME_USED, timeUsed);
for(final String right : rights) {
entry.setProperty(right, rightsSet.get(right));
for(final AccessRight right : rights) {
entry.setProperty(right.toString(), rightsSet.get(right));
}
} catch (final Exception e) {

6
htroot/api/bookmarks/get_bookmarks.java
Unescape View File

@ -58,7 +58,7 @@ public class get_bookmarks {
prop.clear();
sb = (Switchboard) env;
user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT));
isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name
final String username;
@ -239,7 +239,9 @@ public class get_bookmarks {
count++;
}
root = root.replaceAll("(/.[^/]*$)", "");
if ("".equals(root)) root = "/";
if ("".equals(root)) {
root = "/";
}
count = recurseFolders(it, root, count, false, fn);
}
return count;

2
htroot/api/bookmarks/get_folders.java
Unescape View File

@ -23,7 +23,7 @@ public class get_folders {
prop.clear();
sb = (Switchboard) env;
user = sb.userDB.getUser(header);
isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT));
isAdmin = (sb.verifyAuthentication(header, true) || user != null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT));
// set user name
final String username;

2
htroot/api/ymarks/add_ymark.java
Unescape View File

@ -19,7 +19,7 @@ public class add_ymark {
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT);
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);

2
htroot/api/ymarks/delete_ymark.java
Unescape View File

@ -20,7 +20,7 @@ public class delete_ymark {
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT);
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);

2
htroot/api/ymarks/get_treeview.java
Unescape View File

@ -38,7 +38,7 @@ public class get_treeview {
prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT);
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {

2
htroot/api/ymarks/get_xbel.java
Unescape View File

@ -26,7 +26,7 @@ public class get_xbel {
final StringBuilder buffer = new StringBuilder(250);
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT);
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
final String bmk_user;
if(isAdmin || isAuthUser) {

2
htroot/api/ymarks/get_ymark.java
Unescape View File

@ -27,7 +27,7 @@ public class get_ymark {
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT);
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
final TreeSet<String> bookmarks = new TreeSet<String>();
if(isAdmin || isAuthUser) {

2
htroot/api/ymarks/import_ymark.java
Unescape View File

@ -30,7 +30,7 @@ public class import_ymark {
final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.Entry.BOOKMARK_RIGHT);
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
Thread t;
HashMap<String,String> bmk;
ByteArrayInputStream byteIn = null;

171
htroot/yacysearch.java
Unescape View File

@ -62,6 +62,7 @@ import net.yacy.kelondro.util.ISO639;
import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.CrawlProfile.CacheStrategy;
import de.anomic.data.DidYouMean;
import de.anomic.data.UserDB;
import de.anomic.search.AccessTracker;
import de.anomic.search.ContentDomain;
import de.anomic.search.QueryParams;
@ -88,7 +89,11 @@ public class yacysearch {
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header, false);
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
boolean authenticated = sb.adminAuthenticated(header) >= 2;
if (!authenticated) {
final UserDB.Entry user = sb.userDB.getUser(header);
authenticated = (user != null && user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT));
}
final boolean localhostAccess = sb.accessFromLocalhost(header);
final String promoteSearchPageGreeting =
(env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ?
@ -178,7 +183,9 @@ public class yacysearch {
int offset = (newsearch) ? 0 : post.getInt("startRecord", post.getInt("offset", 0));
final int newcount;
if ( authenticated && (newcount = post.getInt("count", 0)) > 0 ) sb.setConfig(SwitchboardConstants.SEARCH_ITEMS, newcount); // set new default maximumRecords if search with "more options"
if ( authenticated && (newcount = post.getInt("count", 0)) > 0 ) {
sb.setConfig(SwitchboardConstants.SEARCH_ITEMS, newcount);
} // set new default maximumRecords if search with "more options"
boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0;
final boolean indexof = (post != null && post.get("indexof","").equals("on"));
@ -211,7 +218,9 @@ public class yacysearch {
final boolean clustersearch = sb.isRobinsonMode() &&
(sb.getConfig("cluster.mode", "").equals("privatecluster") ||
sb.getConfig("cluster.mode", "").equals("publiccluster"));
if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes
if (clustersearch) {
global = true;
} // switches search on, but search target is limited to cluster nodes
// increase search statistic counter
if (!global) {
@ -232,15 +241,21 @@ public class yacysearch {
final ContentDomain contentdom = ContentDomain.contentdomParser(post == null ? "text" : post.get("contentdom", "text"));
// patch until better search profiles are available
if ((contentdom != ContentDomain.TEXT) && (itemsPerPage <= 32)) itemsPerPage = 64;
if ((contentdom != ContentDomain.TEXT) && (itemsPerPage <= 32)) {
itemsPerPage = 64;
}
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
if (trackerHandles == null) {
trackerHandles = new TreeSet<Long>();
}
boolean block = false;
if (Domains.matchesList(client, sb.networkBlacklist)) {
global = false;
if (snippetFetchStrategy != null) snippetFetchStrategy = null;
if (snippetFetchStrategy != null) {
snippetFetchStrategy = null;
}
block = true;
Log.logWarning("LOCAL_SEARCH", "ACCESS CONTROL: BLACKLISTED CLIENT FROM " + client + " gets no permission to search");
} else if (Domains.matchesList(client, sb.networkWhitelist)) {
@ -294,7 +309,9 @@ public class yacysearch {
}
if (querystring.indexOf("/location") >= 0) {
querystring = querystring.replace("/location", "");
if (constraint == null) constraint = new Bitfield(4);
if (constraint == null) {
constraint = new Bitfield(4);
}
constraint.set(Condenser.flag_cat_haslocation, true);
}
int lrp = querystring.indexOf("/language/");
@ -310,19 +327,25 @@ public class yacysearch {
final int inurl = querystring.indexOf("inurl:");
if (inurl >= 0) {
int ftb = querystring.indexOf(' ', inurl);
if (ftb == -1) ftb = querystring.length();
if (ftb == -1) {
ftb = querystring.length();
}
String urlstr = querystring.substring(inurl + 6, ftb);
querystring = querystring.replace("inurl:" + urlstr, "");
if(urlstr.length() > 0) urlmask = ".*" + urlstr + ".*";
if (!urlstr.isEmpty()) {
urlmask = ".*" + urlstr + ".*";
}
}
final int filetype = querystring.indexOf("filetype:");
if (filetype >= 0) {
int ftb = querystring.indexOf(' ', filetype);
if (ftb == -1) ftb = querystring.length();
if (ftb == -1) {
ftb = querystring.length();
}
String ft = querystring.substring(filetype + 9, ftb);
querystring = querystring.replace("filetype:" + ft, "");
while (ft.length() > 0 && ft.charAt(0) == '.') ft = ft.substring(1);
if (ft.length() > 0) {
while (!ft.isEmpty() && ft.charAt(0) == '.') ft = ft.substring(1);
if (!ft.isEmpty()) {
if (urlmask == null) {
urlmask = ".*\\." + ft;
} else {
@ -333,9 +356,13 @@ public class yacysearch {
String tenant = null;
if (post.containsKey("tenant")) {
tenant = post.get("tenant");
if (tenant != null && tenant.length() == 0) tenant = null;
if (tenant != null && tenant.isEmpty()) {
tenant = null;
}
if (tenant != null) {
if (urlmask == null) urlmask = ".*" + tenant + ".*"; else urlmask = ".*" + tenant + urlmask;
if (urlmask == null) {
urlmask = ".*" + tenant + ".*";
} else urlmask = ".*" + tenant + urlmask;
}
}
int site = querystring.indexOf("site:");
@ -343,11 +370,17 @@ public class yacysearch {
String sitehost = null;
if (site >= 0) {
int ftb = querystring.indexOf(' ', site);
if (ftb == -1) ftb = querystring.length();
if (ftb == -1) {
ftb = querystring.length();
}
sitehost = querystring.substring(site + 5, ftb);
querystring = querystring.replace("site:" + sitehost, "");
while (sitehost.length() > 0 && sitehost.charAt(0) == '.') sitehost = sitehost.substring(1);
while (sitehost.endsWith(".")) sitehost = sitehost.substring(0, sitehost.length() - 1);
while (sitehost.length() > 0 && sitehost.charAt(0) == '.') {
sitehost = sitehost.substring(1);
}
while (sitehost.endsWith(".")) {
sitehost = sitehost.substring(0, sitehost.length() - 1);
}
sitehash = DigestURI.domhash(sitehost);
}
@ -369,12 +402,16 @@ public class yacysearch {
String author;
if (quotes) {
int ftb = querystring.indexOf((char) 39, authori + 8);
if (ftb == -1) ftb = querystring.length() + 1;
if (ftb == -1) {
ftb = querystring.length() + 1;
}
author = querystring.substring(authori + 8, ftb);
querystring = querystring.replace("author:'" + author + "'", "");
} else {
int ftb = querystring.indexOf(' ', authori);
if (ftb == -1) ftb = querystring.length();
if (ftb == -1) {
ftb = querystring.length();
}
author = querystring.substring(authori + 7, ftb);
querystring = querystring.replace("author:" + author, "");
}
@ -383,31 +420,41 @@ public class yacysearch {
final int tld = querystring.indexOf("tld:");
if (tld >= 0) {
int ftb = querystring.indexOf(' ', tld);
if (ftb == -1) ftb = querystring.length();
if (ftb == -1) {
ftb = querystring.length();
}
String domain = querystring.substring(tld + 4, ftb);
querystring = querystring.replace("tld:" + domain, "");
while (domain.length() > 0 && domain.charAt(0) == '.') domain = domain.substring(1);
if (domain.indexOf('.') < 0) domain = "\\." + domain; // is tld
while (domain.length() > 0 && domain.charAt(0) == '.') {
domain = domain.substring(1);
}
if (domain.indexOf('.') < 0) {
domain = "\\." + domain;
} // is tld
if (domain.length() > 0) {
if (urlmask == null) {
urlmask = "[a-zA-Z]*://[^/]*" + domain + "/.*";
} else {
urlmask = "[a-zA-Z]*://[^/]*" + domain + "/.*" + urlmask;
}
urlmask = "[a-zA-Z]*://[^/]*" + domain + "/.*" + ((urlmask != null) ? urlmask : "");
}
}
if (urlmask == null || urlmask.length() == 0) urlmask = originalUrlMask; //if no urlmask was given
if (urlmask == null || urlmask.isEmpty()) {
urlmask = originalUrlMask;
} //if no urlmask was given
// read the language from the language-restrict option 'lr'
// if no one is given, use the user agent or the system language as default
String language = (post == null) ? lr : post.get("lr", lr);
if (language.startsWith("lang_")) language = language.substring(5);
if (language.startsWith("lang_")) {
language = language.substring(5);
}
if (!ISO639.exists(language)) {
// find out language of the user by reading of the user-agent string
String agent = header.get(HeaderFramework.ACCEPT_LANGUAGE);
if (agent == null) agent = System.getProperty("user.language");
if (agent == null) {
agent = System.getProperty("user.language");
}
language = (agent == null) ? "en" : ISO639.userAgentLanguageDetection(agent);
if (language == null) language = "en";
if (language == null) {
language = "en";
}
}
// navigation
@ -416,7 +463,7 @@ public class yacysearch {
// the query
final TreeSet<String>[] query = QueryParams.cleanQuery(querystring.trim()); // converts also umlaute
int maxDistance = (querystring.indexOf('"') >= 0) ? maxDistance = query.length - 1 : Integer.MAX_VALUE;
int maxDistance = (querystring.indexOf('"') >= 0) ? query.length - 1 : Integer.MAX_VALUE;
// filter out stopwords
final SortedSet<String> filtered = SetTools.joinConstructive(query[0], Switchboard.stopwords);
@ -425,26 +472,28 @@ public class yacysearch {
}
// if a minus-button was hit, remove a special reference first
if (post != null && post.containsKey("deleteref")) try {
if (!sb.verifyAuthentication(header, true)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
// delete the index entry locally
final String delHash = post.get("deleteref", ""); // urlhash
indexSegment.termIndex().remove(Word.words2hashesHandles(query[0]), delHash.getBytes());
if (post != null && post.containsKey("deleteref")) {
try {
if (!sb.verifyAuthentication(header, true)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
// make new news message with negative voting
if (!sb.isRobinsonMode()) {
final Map<String, String> map = new HashMap<String, String>();
map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
// delete the index entry locally
final String delHash = post.get("deleteref", ""); // urlhash
indexSegment.termIndex().remove(Word.words2hashesHandles(query[0]), delHash.getBytes());
// make new news message with negative voting
if (!sb.isRobinsonMode()) {
final Map<String, String> map = new HashMap<String, String>();
map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
} catch (IOException e) {
Log.logException(e);
}
} catch (IOException e) {
Log.logException(e);
}
// if a plus-button was hit, create new voting message
@ -478,7 +527,7 @@ public class yacysearch {
}
// prepare search properties
final boolean globalsearch = (global) && indexReceiveGranted; /* && (yacyonline)*/
final boolean globalsearch = (global) && indexReceiveGranted;
// do the search
final HandleSet queryHashes = Word.words2hashesHandles(query[0]);
@ -561,9 +610,15 @@ public class yacysearch {
} catch (InterruptedException e1) {} // wait a little time to get first results in the search
if (offset == 0) {
if (sitehost != null && sb.getConfigBool("heuristic.site", false) && authenticated) sb.heuristicSite(theSearch, sitehost);
if ((heuristicScroogle >= 0 || sb.getConfigBool("heuristic.scroogle", false)) && authenticated) sb.heuristicScroogle(theSearch);
if ((heuristicBlekko >= 0 || sb.getConfigBool("heuristic.blekko", false)) && authenticated) sb.heuristicRSS("http://blekko.com/ws/$+/rss", theSearch, "blekko");
if (sitehost != null && sb.getConfigBool("heuristic.site", false) && authenticated) {
sb.heuristicSite(theSearch, sitehost);
}
if ((heuristicScroogle >= 0 || sb.getConfigBool("heuristic.scroogle", false)) && authenticated) {
sb.heuristicScroogle(theSearch);
}
if ((heuristicBlekko >= 0 || sb.getConfigBool("heuristic.blekko", false)) && authenticated) {
sb.heuristicRSS("http://blekko.com/ws/$+/rss", theSearch, "blekko");
}
}
// log
@ -631,7 +686,9 @@ public class yacysearch {
}
}
sb.localSearchTracker.put(client, trackerHandles);
if (sb.localSearchTracker.size() > 1000) sb.localSearchTracker.remove(sb.localSearchTracker.keys().nextElement());
if (sb.localSearchTracker.size() > 1000) {
sb.localSearchTracker.remove(sb.localSearchTracker.keys().nextElement());
}
} catch (Exception e) {
Log.logException(e);
}
@ -721,7 +778,9 @@ public class yacysearch {
// adding some additional properties needed for the rss feed
String hostName = header.get("Host", "localhost");
if (hostName.indexOf(':') == -1) hostName += ":" + serverCore.getPortNr(env.getConfig("port", "8090"));
if (hostName.indexOf(':') == -1) {
hostName += ":" + serverCore.getPortNr(env.getConfig("port", "8090"));
}
prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html");
prop.put("rssYacyImageURL", "http://" + hostName + "/env/grafics/yacy.gif");
}

298
source/de/anomic/data/UserDB.java
Unescape View File

@ -1,12 +1,13 @@
//userDB.java
//UserDB.java
//-------------------------------------
//part of YACY
//
//(C) 2005, 2006 by Martin Thelian
// Alexander Schier
//
//last change: $LastChangedDate$ by $LastChangedBy$
//Revision: $LastChangedRevision$
//$LastChangedDate$
//$LastChangedRevision$
//$LastChangedBy$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General Public License as published by
@ -60,13 +61,14 @@ public final class UserDB {
public UserDB(final File userTableFile) throws IOException {
this.userTableFile = userTableFile;
userTableFile.getParentFile().mkdirs();
//this.userTable = new MapView(BLOBTree.toHeap(userTableFile, true, true, 128, 256, '_', NaturalOrder.naturalOrder, userTableFile), 10, '_');
this.userTable = new MapHeap(userTableFile, 128, NaturalOrder.naturalOrder, 1024 * 64, 10, '_');
}
void resetDatabase() {
// deletes the database and creates a new one
if (userTable != null) userTable.close();
if (userTable != null) {
userTable.close();
}
FileUtils.deletedelete(userTableFile);
userTableFile.getParentFile().mkdirs();
try {
@ -87,25 +89,25 @@ public final class UserDB {
public void removeEntry(final String hostName) {
try {
userTable.delete(hostName.toLowerCase().getBytes());
} catch (final IOException e) {}
} catch (final IOException e) {
Log.logException(e);
}
}
public Entry getEntry(String userName) {
if(userName.length()>128){
userName=userName.substring(0, 127);
if (userName.length() > 128) {
userName = userName.substring(0, 127);
}
Map<String, String> record;
Map<String, String> record = null;
try {
record = userTable.get(userName.getBytes());
} catch (final IOException e) {
Log.logException(e);
return null;
} catch (RowSpaceExceededException e) {
Log.logException(e);
return null;
}
if (record == null) return null;
return new Entry(userName, record);
return (record != null) ? new Entry(userName, record) : null;
}
public Entry createEntry(final String userName, final Map<String, String> userProps) throws IllegalArgumentException{
@ -124,18 +126,17 @@ public final class UserDB {
}
/**
* use a ProxyAuth String to authenticate a user
* @param auth a base64 Encoded String, which contains "username:pw".
* Use a ProxyAuth String to authenticate user.
* @param auth base64 Encoded String, which contains "username:pw".
*/
public Entry proxyAuth(final String auth) {
if(auth == null) {
return null;
}
Entry entry = null;
final String[] tmp= Base64Order.standardCoder.decodeString(auth.trim().substring(6)).split(":");
if(tmp.length == 2){
entry = this.passwordAuth(tmp[0], tmp[1]);
if (auth != null) {
final String[] tmp = Base64Order.standardCoder.decodeString(auth.trim().substring(6)).split(":");
if (tmp.length == 2) {
entry = this.passwordAuth(tmp[0], tmp[1]);
}
}
return entry;
}
@ -154,49 +155,44 @@ public final class UserDB {
}
return entry;
}
/**
* determinate, if a user has Adminrights from a authorisation http-headerfield it tests both userDB and oldstyle adminpw.
* Determine if a user has admin rights from a authorisation http-headerfield.
* Tests both userDB and old style adminpw.
*
* @param auth
* the http-headerline for authorisation
* @param auth http-headerline for authorisation.
* @param cookies
*/
public boolean hasAdminRight(final String auth, final String cookies) {
final Entry entry = getUser(auth, cookies);
if (entry != null) {
return entry.hasRight(Entry.ADMIN_RIGHT);
}
// else if(entry != null && cookieAdminAuth(cookies))
// return entry.hasAdminRight();
else {
return false;
}
return (entry != null) ? entry.hasRight(AccessRight.ADMIN_RIGHT) : false;
}
/**
* use a ProxyAuth String to authenticate a user and save the ip/username for ipAuth
* @param auth a base64 Encoded String, which contains "username:pw".
* @param ip an ip.
* Use ProxyAuth String to authenticate user and save IP/username for ipAuth.
* @param auth base64 Encoded String, which contains "username:pw".
* @param ip IP address.
*/
public Entry proxyAuth(final String auth, final String ip) {
final Entry entry=proxyAuth(auth);
if(entry == null){
return null;
final Entry entry = proxyAuth(auth);
if (entry != null) {
entry.updateLastAccess(false);
this.ipUsers.put(ip, entry.getUserName());
}
entry.updateLastAccess(false);
this.ipUsers.put(ip, entry.getUserName());
return entry;
}
/**
* authenticate a user by ip, if he had used proxyAuth in the last 10 Minutes
* @param ip the IP of the User
* Authenticate a user by ip, if he has used proxyAuth in the last 10 minutes.
* @param ip IP address of user.
*/
public Entry ipAuth(final String ip) {
if(this.ipUsers.containsKey(ip)){
final String user=this.ipUsers.get(ip);
final Entry entry=this.getEntry(user);
final Long entryTimestamp=entry.getLastAccess();
if(entryTimestamp == null || (System.currentTimeMillis()-entryTimestamp.longValue()) > (1000*60*10) ){ //no timestamp or older than 10 Minutes
final String user = this.ipUsers.get(ip);
final Entry entry = this.getEntry(user);
final Long entryTimestamp = entry.getLastAccess();
if (entryTimestamp == null ||
(System.currentTimeMillis()-entryTimestamp.longValue()) > (1000*60*10) ){ //no timestamp or older than 10 Minutes
return null;
}
return entry; //All OK
@ -222,17 +218,16 @@ public final class UserDB {
}
public Entry passwordAuth(final String user, final String password, final String ip){
final Entry entry=passwordAuth(user, password);
if(entry == null){
return null;
final Entry entry = passwordAuth(user, password);
if (entry != null) {
entry.updateLastAccess(false);
this.ipUsers.put(ip, entry.getUserName()); //XXX: This is insecure. TODO: use cookieauth
}
entry.updateLastAccess(false);
this.ipUsers.put(ip, entry.getUserName()); //XXX: This is insecure. TODO: use cookieauth
return entry;
}
public Entry md5Auth(final String user, final String md5) {
final Entry entry=this.getEntry(user);
final Entry entry = this.getEntry(user);
if (entry != null && entry.getMD5EncodedUserPwd().equals(md5)) {
if (entry.isLoggedOut()){
try {
@ -284,7 +279,7 @@ public final class UserDB {
public static String getLoginToken(final String cookies){
final String[] cookie = cookies.split(";"); //TODO: Mozilla uses "; "
for (String c :cookie) {
for (final String c :cookie) {
String[] pair = c.split("=");
if (pair[0].trim().equals("login")) {
return pair[1].trim();
@ -294,12 +289,42 @@ public final class UserDB {
}
public void adminLogout(final String logintoken){
if(cookieUsers.containsKey(logintoken)){
if (cookieUsers.containsKey(logintoken)) {
//XXX: We could check, if its == "admin", but we want to logout anyway.
cookieUsers.remove(logintoken);
}
}
public enum AccessRight {
//to create new rights, you just add them here
UPLOAD_RIGHT("uploadRight", "Upload"),
DOWNLOAD_RIGHT("downloadRight", "Download"),
ADMIN_RIGHT("adminRight", "Admin"),
PROXY_RIGHT("proxyRight", "Proxy usage"),
BLOG_RIGHT("blogRight", "Blog"),
WIKIADMIN_RIGHT("wikiAdminRight", "Wiki Admin"),
BOOKMARK_RIGHT("bookmarkRight", "Bookmark"),
EXTENDED_SEARCH_RIGHT("extendedSearchRight", "Extended Search");
private String name;
private String friendlyName;
AccessRight(final String name, final String friendlyName) {
this.friendlyName = friendlyName;
this.name = name;
}
@Override
public String toString() {
return name;
}
public String getFriendlyName() {
return friendlyName;
}
}
public class Entry {
public static final String MD5ENCODED_USERPWD_STRING = "MD5_user:pwd";
@ -313,20 +338,7 @@ public final class UserDB {
public static final String TIME_LIMIT = "timeLimit";
public static final String TRAFFIC_SIZE = "trafficSize";
public static final String TRAFFIC_LIMIT = "trafficLimit";
public static final String UPLOAD_RIGHT = "uploadRight";
public static final String DOWNLOAD_RIGHT = "downloadRight";
public static final String ADMIN_RIGHT = "adminRight";
public static final String PROXY_RIGHT = "proxyRight";
public static final String BLOG_RIGHT = "blogRight";
public static final String WIKIADMIN_RIGHT = "wikiAdminRight";
public static final String BOOKMARK_RIGHT = "bookmarkRight";
//to create new rights, you just need to edit this strings
public static final String RIGHT_TYPES=
ADMIN_RIGHT+","+DOWNLOAD_RIGHT+","+UPLOAD_RIGHT+","+PROXY_RIGHT+","+
BLOG_RIGHT+","+BOOKMARK_RIGHT+","+WIKIADMIN_RIGHT;
public static final String RIGHT_NAMES="Admin,Download,Upload,Proxy usage,Blog,Bookmark,Wiki Admin,SOAP";
public static final int PROXY_ALLOK = 0; //can Surf
public static final int PROXY_ERROR = 1; //unknown error
public static final int PROXY_NORIGHT = 2; //no proxy right
@ -338,18 +350,20 @@ public final class UserDB {
private final Calendar oldDate, newDate;
public Entry(final String userName, final Map<String, String> mem) throws IllegalArgumentException {
if ((userName == null) || (userName.length() == 0))
if ((userName == null) || (userName.isEmpty())) {
throw new IllegalArgumentException("Username needed.");
if(userName.length()>128){
}
if (userName.length()>128) {
throw new IllegalArgumentException("Username too long!");
}
this.userName = userName.trim();
if (this.userName.length() < USERNAME_MIN_LENGTH)
if (this.userName.length() < USERNAME_MIN_LENGTH) {
throw new IllegalArgumentException("Username too short. Length should be >= " + USERNAME_MIN_LENGTH);
if (mem == null) this.mem = new HashMap<String, String>();
else this.mem = mem;
}
this.mem = (mem == null) ? new HashMap<String, String>() : mem;
if (mem == null || !mem.containsKey(AUTHENTICATION_METHOD)) {
this.mem.put(AUTHENTICATION_METHOD,"yacy");
@ -363,67 +377,79 @@ public final class UserDB {
}
public String getFirstName() {
return (this.mem.containsKey(USER_FIRSTNAME)?this.mem.get(USER_FIRSTNAME):null);
return (this.mem.containsKey(USER_FIRSTNAME) ? this.mem.get(USER_FIRSTNAME) : null);
}
public String getLastName() {
return (this.mem.containsKey(USER_LASTNAME)?this.mem.get(USER_LASTNAME):null);
return (this.mem.containsKey(USER_LASTNAME) ? this.mem.get(USER_LASTNAME) : null);
}
public String getAddress() {
return (this.mem.containsKey(USER_ADDRESS)?this.mem.get(USER_ADDRESS):null);
return (this.mem.containsKey(USER_ADDRESS) ? this.mem.get(USER_ADDRESS) : null);
}
public long getTimeUsed() {
long ret = 0L;
if (this.mem.containsKey(TIME_USED)) {
try{
return Long.parseLong(this.mem.get(TIME_USED));
}catch(final NumberFormatException e){
return 0;
ret = Long.parseLong(this.mem.get(TIME_USED));
} catch (final NumberFormatException e){
Log.logException(e);
}
} else {
try {
this.setProperty(TIME_USED,"0");
} catch (final Exception e) {
Log.logException(e);
}
}
try {
this.setProperty(TIME_USED,"0");
} catch (final Exception e) {
Log.logException(e);
}
return 0;
return ret;
}
public long getTimeLimit() {
long ret = 0L;
if (this.mem.containsKey(TIME_LIMIT)) {
try{
return Long.parseLong(this.mem.get(TIME_LIMIT));
}catch(final NumberFormatException e){
return 0;
try {
ret = Long.parseLong(this.mem.get(TIME_LIMIT));
} catch (final NumberFormatException e){
Log.logException(e);
}
} else {
try {
this.setProperty(TIME_LIMIT,"0");
} catch (final Exception e) {
Log.logException(e);
}
}
try {
this.setProperty(TIME_LIMIT,"0");
} catch (final Exception e) {
Log.logException(e);
}
return 0;
return ret;
}
public long getTrafficSize() {
long ret = 0L;
if (this.mem.containsKey(TRAFFIC_SIZE)) {
return Long.parseLong(this.mem.get(TRAFFIC_SIZE));
}
try {
this.setProperty(TRAFFIC_SIZE,"0");
} catch (final Exception e) {
Log.logException(e);
try {
ret = Long.parseLong(this.mem.get(TRAFFIC_SIZE));
} catch (final NumberFormatException e) {
Log.logException(e);
}
} else {
try {
this.setProperty(TRAFFIC_SIZE,"0");
} catch (final Exception e) {
Log.logException(e);
}
}
return 0;
return ret;
}
public Long getTrafficLimit() {
return (this.mem.containsKey(TRAFFIC_LIMIT)?Long.valueOf(this.mem.get(TRAFFIC_LIMIT)):null);
return (this.mem.containsKey(TRAFFIC_LIMIT) ? Long.valueOf(this.mem.get(TRAFFIC_LIMIT)) : null);
}
public long updateTrafficSize(final long responseSize) {
if (responseSize < 0) throw new IllegalArgumentException("responseSize must be greater or equal zero.");
if (responseSize < 0) {
throw new IllegalArgumentException("responseSize must be greater or equal zero.");
}
final long currentTrafficSize = getTrafficSize();
final long newTrafficSize = currentTrafficSize + responseSize;
@ -436,22 +462,25 @@ public final class UserDB {
}
public Long getLastAccess() {
return (this.mem.containsKey(LAST_ACCESS)?Long.valueOf(this.mem.get(LAST_ACCESS)):null);
return (this.mem.containsKey(LAST_ACCESS) ? Long.valueOf(this.mem.get(LAST_ACCESS)) : null);
}
public int surfRight(){
final long timeUsed=this.updateLastAccess(true);
if (!this.hasRight(Entry.PROXY_RIGHT))
return PROXY_NORIGHT;
final int ret;
if(! (this.getTimeLimit() <= 0 || ( timeUsed < this.getTimeLimit())) ){ //no timelimit or timelimit not reached
return PROXY_TIMELIMIT_REACHED;
if (!this.hasRight(AccessRight.PROXY_RIGHT)) {
ret = PROXY_NORIGHT;
} else if(! (this.getTimeLimit() <= 0 || (timeUsed < this.getTimeLimit())) ){ //no timelimit or timelimit not reached
ret = PROXY_TIMELIMIT_REACHED;
} else {
ret = PROXY_ALLOK;
}
return PROXY_ALLOK;
return ret;
}
public boolean canSurf(){
return (this.surfRight() == PROXY_ALLOK) ? true : false;
return (this.surfRight() == PROXY_ALLOK);
}
public long updateLastAccess(final boolean incrementTimeUsed) {
@ -459,16 +488,17 @@ public final class UserDB {
}
public long updateLastAccess(final long timeStamp, final boolean incrementTimeUsed) {
if (timeStamp < 0) throw new IllegalArgumentException();
if (timeStamp < 0) {
throw new IllegalArgumentException();
}
final Long lastAccess = this.getLastAccess();
long newTimeUsed = getTimeUsed();
if (incrementTimeUsed) {
if ((lastAccess == null)||((lastAccess != null)&&(timeStamp-lastAccess.longValue()>=1000*60))) { //1 minute
//this.mem.put(TIME_USED,Long.toString(newTimeUsed = ++oldTimeUsed));
if ((lastAccess == null) || (timeStamp - lastAccess.longValue() >= 1000*60)) {//1 minute
newTimeUsed++;
if(lastAccess != null){
if (lastAccess != null) {
this.oldDate.setTime(new Date(lastAccess.longValue()));
this.newDate.setTime(new Date(System.currentTimeMillis()));
if(
@ -497,7 +527,7 @@ public final class UserDB {
}
public String getMD5EncodedUserPwd() {
return (this.mem.containsKey(MD5ENCODED_USERPWD_STRING)?this.mem.get(MD5ENCODED_USERPWD_STRING):null);
return (this.mem.containsKey(MD5ENCODED_USERPWD_STRING)) ? this.mem.get(MD5ENCODED_USERPWD_STRING) : null;
}
public Map<String, String> getProperties() {
@ -510,63 +540,71 @@ public final class UserDB {
}
public String getProperty(final String propName, final String defaultValue) {
return (this.mem.containsKey(propName)?this.mem.get(propName):defaultValue);
return (this.mem.containsKey(propName) ? this.mem.get(propName) : defaultValue);
}
public boolean hasRight(final String rightName){
return (this.mem.containsKey(rightName)?this.mem.get(rightName).equals("true"):false);
public boolean hasRight(final AccessRight accessRight){
return (this.mem.containsKey(accessRight.toString())) ? this.mem.get(accessRight.toString()).equals("true") : false;
}
/**
* @deprecated use hasRight(UPLOAD_RIGHT) instead
*/
@Deprecated
public boolean hasUploadRight() {
return this.hasRight(UPLOAD_RIGHT);
return this.hasRight(AccessRight.UPLOAD_RIGHT);
}
/**
* @deprecated use hasRight(DOWNLOAD_RIGHT) instead
*/
@Deprecated
public boolean hasDownloadRight() {
return this.hasRight(DOWNLOAD_RIGHT);
return this.hasRight(AccessRight.DOWNLOAD_RIGHT);
}
/**
* @deprecated use hasRight(PROXY_RIGHT) instead
*/
@Deprecated
public boolean hasProxyRight() {
return this.hasRight(PROXY_RIGHT);
return this.hasRight(AccessRight.PROXY_RIGHT);
}
/**
* @deprecated use hasRight(ADMIN_RIGHT) instead
*/
@Deprecated
public boolean hasAdminRight() {
return this.hasRight(ADMIN_RIGHT);
return this.hasRight(AccessRight.ADMIN_RIGHT);
}
/**
* @deprecated use hasRight(BLOG_RIGHT) instead
*/
@Deprecated
public boolean hasBlogRight() {
return this.hasRight(BLOG_RIGHT);
return this.hasRight(AccessRight.BLOG_RIGHT);
}
/**
* @deprecated use hasRight(WIKIADMIN_RIGHT) instead
*/
@Deprecated
public boolean hasWikiAdminRight() {
return this.hasRight(WIKIADMIN_RIGHT);
return this.hasRight(AccessRight.WIKIADMIN_RIGHT);
}
/**
* @deprecated use hasRight(BOOKMARK_RIGHT) instead
*/
@Deprecated
public boolean hasBookmarkRight() {
return this.hasRight(BOOKMARK_RIGHT);
return this.hasRight(AccessRight.BOOKMARK_RIGHT);
}
public boolean isLoggedOut(){
return (this.mem.containsKey(LOGGED_OUT)?this.mem.get(LOGGED_OUT).equals("true"):false);
return (this.mem.containsKey(LOGGED_OUT) ? this.mem.get(LOGGED_OUT).equals("true") : false);
}
public void logout(final String ip, final String logintoken){
@ -595,7 +633,7 @@ public final class UserDB {
public String toString() {
final StringBuilder str = new StringBuilder();
str.append((this.userName==null)?"null":this.userName).append(": ");
str.append((this.userName == null) ? "null" : this.userName).append(": ");
if (this.mem != null) {
str.append(this.mem.toString());

298
source/de/anomic/search/Switchboard.java
Unescape View File

@ -258,8 +258,9 @@ public final class Switchboard extends serverSwitch {
yacySeed.ANON_PREFIX = getConfig("peernameprefix", "_anon");
// UPnP port mapping
if (getConfigBool(SwitchboardConstants.UPNP_ENABLED, false))
if (getConfigBool(SwitchboardConstants.UPNP_ENABLED, false)) {
InstantBusyThread.oneTimeJob(UPnP.class, "addPortMapping", UPnP.log, 0);
}
// init TrayIcon if possible
Tray.init(this);
@ -269,9 +270,13 @@ public final class Switchboard extends serverSwitch {
// memory configuration
this.useTailCache = getConfigBool("ramcopy", true);
if (MemoryControl.available() > 1024 * 1024 * 1024 * 1) this.useTailCache = true;
if (MemoryControl.available() > 1024 * 1024 * 1024 * 1) {
this.useTailCache = true;
}
this.exceed134217727 = getConfigBool("exceed134217727", true);
if (MemoryControl.available() > 1024 * 1024 * 1024 * 2) this.exceed134217727 = true;
if (MemoryControl.available() > 1024 * 1024 * 1024 * 2) {
this.exceed134217727 = true;
}
// load values from configs
final File indexPath = getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, SwitchboardConstants.INDEX_PATH_DEFAULT);
@ -296,6 +301,7 @@ public final class Switchboard extends serverSwitch {
// init libraries
this.log.logConfig("initializing libraries");
new Thread() {
@Override
public void run() {
LibraryProvider.initialize(dictionariesPath);
}
@ -385,7 +391,9 @@ public final class Switchboard extends serverSwitch {
this.webStructure = new WebStructureGraph(log, new File(queuesRoot, "webStructure.map"));
// configuring list path
if (!(listsPath.exists())) listsPath.mkdirs();
if (!(listsPath.exists())) {
listsPath.mkdirs();
}
// load coloured lists
if (blueList == null) {
@ -480,6 +488,7 @@ public final class Switchboard extends serverSwitch {
// Init bookmarks DB: needs more time since this does a DNS lookup for each Bookmark.
// Can be started concurrently
new Thread(){
@Override
public void run() {
try {
initBookmarks();
@ -729,10 +738,11 @@ public final class Switchboard extends serverSwitch {
String k;
while (ki.hasNext()) {
k = ki.next();
if (k.startsWith("network.unit.update.location")) d.add(k);
if (k.startsWith("network.unit.bootstrap")) d.add(k);
if (k.startsWith("network.unit.update.location") || k.startsWith("network.unit.bootstrap")) {
d.add(k);
}
}
for (String s:d) this.removeConfig(s); // must be removed afterwards otherwise a ki.remove() would not remove the property on file
for (final String s : d) this.removeConfig(s); // must be removed afterwards otherwise a ki.remove() would not remove the property on file
// include additional network definition properties into our settings
// note that these properties cannot be set in the application because they are
@ -751,7 +761,9 @@ public final class Switchboard extends serverSwitch {
cryptoLib = new CryptoLib();
while (true) {
String location = getConfig("network.unit.update.location" + i, "");
if (location.length() == 0) break;
if (location.isEmpty()) {
break;
}
DigestURI locationURL;
try {
// try to parse url
@ -762,9 +774,9 @@ public final class Switchboard extends serverSwitch {
PublicKey publicKey = null;
// get public key if it's in config
try {
String publicKeyString = getConfig("network.unit.update.location" + i + ".key", null);
final String publicKeyString = getConfig("network.unit.update.location" + i + ".key", null);
if (publicKeyString != null) {
byte[] publicKeyBytes = Base64Order.standardCoder.decode(publicKeyString.trim());
final byte[] publicKeyBytes = Base64Order.standardCoder.decode(publicKeyString.trim());
publicKey = cryptoLib.getPublicKeyFromBytes(publicKeyBytes);
}
} catch (InvalidKeySpecException e) {
@ -812,9 +824,13 @@ public final class Switchboard extends serverSwitch {
log.logInfo("SWITCH NETWORK: switching to '" + networkDefinition + "'");
// pause crawls
final boolean lcp = crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
if (!lcp) pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
if (!lcp) {
pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
}
final boolean rcp = crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
if (!rcp) pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
if (!rcp) {
pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
}
// trigger online caution
proxyLastAccess = System.currentTimeMillis() + 3000; // at least 3 seconds online caution to prevent unnecessary action on database meanwhile
log.logInfo("SWITCH NETWORK: SHUT DOWN OF OLD INDEX DATABASE...");
@ -1204,7 +1220,9 @@ public final class Switchboard extends serverSwitch {
// get next queue entry and start a queue processing
if (response == null) {
if (this.log.isFine()) log.logFine("deQueue: queue entry is null");
if (this.log.isFine()) {
log.logFine("deQueue: queue entry is null");
}
return "queue entry is null";
}
if (response.profile() == null) {
@ -1250,7 +1268,9 @@ public final class Switchboard extends serverSwitch {
}
// put document into the concurrent processing queue
if (log.isFinest()) log.logFinest("deQueue: passing to indexing queue: " + response.url().toNormalform(true, false));
if (log.isFinest()) {
log.logFinest("deQueue: passing to indexing queue: " + response.url().toNormalform(true, false));
}
try {
this.indexingDocumentProcessor.enQueue(new indexingQueueEntry(Segments.Process.LOCALCRAWLING, response, null, null));
return null;
@ -1292,7 +1312,9 @@ public final class Switchboard extends serverSwitch {
} else {
try {
InputStream is = new BufferedInputStream(new FileInputStream(infile));
if (s.endsWith(".gz")) is = new GZIPInputStream(is);
if (s.endsWith(".gz")) {
is = new GZIPInputStream(is);
}
processSurrogate(is, infile.getName());
} catch (IOException e) {
Log.logException(e);
@ -1307,7 +1329,9 @@ public final class Switchboard extends serverSwitch {
OutputStream os = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile)));
FileUtils.copy(new BufferedInputStream(new FileInputStream(outfile)), os);
os.close();
if (gzfile.exists()) FileUtils.deletedelete(outfile);
if (gzfile.exists()) {
FileUtils.deletedelete(outfile);
}
} catch (FileNotFoundException e) {
Log.logException(e);
} catch (IOException e) {
@ -1368,9 +1392,13 @@ public final class Switchboard extends serverSwitch {
String[] surrogatelist = this.surrogatesInPath.list();
if (surrogatelist.length > 100) return 100;
int count = 0;
for (String s: surrogatelist) {
if (s.endsWith(".xml")) count++;
if (count >= 100) break;
for (final String s: surrogatelist) {
if (s.endsWith(".xml")) {
count++;
}
if (count >= 100) {
break;
}
}
return count;
}
@ -1383,18 +1411,19 @@ public final class Switchboard extends serverSwitch {
// work off fresh entries from the proxy or from the crawler
String cautionCause = onlineCaution();
if (cautionCause != null) {
if (this.log.isFine())
if (this.log.isFine()) {
log.logFine("deQueue: online caution for " + cautionCause
+ ", omitting resource stack processing");
}
return false;
}
try {
// check surrogates
String[] surrogatelist = this.surrogatesInPath.list();
final String[] surrogatelist = this.surrogatesInPath.list();
if (surrogatelist.length > 0) {
// look if the is any xml inside
for (String surrogate: surrogatelist) {
for (final String surrogate: surrogatelist) {
// check for interruption
checkInterruption();
@ -1434,7 +1463,7 @@ public final class Switchboard extends serverSwitch {
int c = 0;
if ((crawlQueues.delegatedURL.stackSize() > 1000)) c++;
if ((crawlQueues.errorURL.stackSize() > 1000)) c++;
for (EventOrigin origin: EventOrigin.values()) {
for (final EventOrigin origin: EventOrigin.values()) {
if (ResultURLs.getStackSize(origin) > 1000) c++;
}
return c;
@ -1444,7 +1473,7 @@ public final class Switchboard extends serverSwitch {
try {
// clear caches if necessary
if (!MemoryControl.request(8000000L, false)) {
for (Segment indexSegment: this.indexSegments) indexSegment.urlMetadata().clearCache();
for (final Segment indexSegment: this.indexSegments) indexSegment.urlMetadata().clearCache();
SearchEventCache.cleanupEvents(true);
}
@ -1458,7 +1487,7 @@ public final class Switchboard extends serverSwitch {
// refresh recrawl dates
try{
CrawlProfile selentry;
for (byte[] handle: crawler.getActive()) {
for (final byte[] handle: crawler.getActive()) {
selentry = crawler.getActive(handle);
assert selentry.handle() != null : "profile.name = " + selentry.name();
if (selentry.handle() == null) {
@ -1471,7 +1500,6 @@ public final class Switchboard extends serverSwitch {
Long.toString(CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_PROXY_RECRAWL_CYCLE)));
insert = true;
}
// if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_REMOTE));
if (selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_TEXT)) {
selentry.put(CrawlProfile.RECRAWL_IF_OLDER,
Long.toString(CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_TEXT_RECRAWL_CYCLE)));
@ -1505,10 +1533,10 @@ public final class Switchboard extends serverSwitch {
// execute scheduled API actions
Tables.Row row;
ArrayList<String> pks = new ArrayList<String>();
Date now = new Date();
final List<String> pks = new ArrayList<String>();
final Date now = new Date();
try {
Iterator<Tables.Row> plainIterator = this.tables.iterator(WorkTables.TABLE_API_NAME);
final Iterator<Tables.Row> plainIterator = this.tables.iterator(WorkTables.TABLE_API_NAME);
final Iterator<Tables.Row> mapIterator = this.tables.orderBy(plainIterator, -1, WorkTables.TABLE_API_COL_DATE_RECORDING).iterator();
while (mapIterator.hasNext()) {
row = mapIterator.next();
@ -1521,7 +1549,7 @@ public final class Switchboard extends serverSwitch {
} catch (IOException e) {
Log.logException(e);
}
for (String pk: pks) try {
for (final String pk: pks) try {
row = this.tables.select(WorkTables.TABLE_API_NAME, pk.getBytes());
WorkTables.calculateAPIScheduler(row, true); // calculate next update time
this.tables.update(WorkTables.TABLE_API_NAME, row);
@ -1532,8 +1560,8 @@ public final class Switchboard extends serverSwitch {
Log.logException(e);
continue;
}
Map<String, Integer> callResult = this.tables.execAPICalls("localhost", (int) this.getConfigLong("port", 8090), this.getConfig("adminAccountBase64MD5", ""), pks);
for (Map.Entry<String, Integer> call: callResult.entrySet()) {
final Map<String, Integer> callResult = this.tables.execAPICalls("localhost", (int) this.getConfigLong("port", 8090), this.getConfig("adminAccountBase64MD5", ""), pks);
for (final Map.Entry<String, Integer> call: callResult.entrySet()) {
log.logInfo("Scheduler executed api call, response " + call.getValue() + ": " + call.getKey());
}
@ -1543,22 +1571,28 @@ public final class Switchboard extends serverSwitch {
// clean up delegated stack
checkInterruption();
if ((crawlQueues.delegatedURL.stackSize() > 1000)) {
if (this.log.isFine()) log.logFine("Cleaning Delegated-URLs report stack, " + crawlQueues.delegatedURL.stackSize() + " entries on stack");
if (this.log.isFine()) {
log.logFine("Cleaning Delegated-URLs report stack, " + crawlQueues.delegatedURL.stackSize() + " entries on stack");
}
crawlQueues.delegatedURL.clearStack();
}
// clean up error stack
checkInterruption();
if ((crawlQueues.errorURL.stackSize() > 1000)) {
if (this.log.isFine()) log.logFine("Cleaning Error-URLs report stack, " + crawlQueues.errorURL.stackSize() + " entries on stack");
if (this.log.isFine()) {
log.logFine("Cleaning Error-URLs report stack, " + crawlQueues.errorURL.stackSize() + " entries on stack");
}
crawlQueues.errorURL.clearStack();
}
// clean up loadedURL stack
for (EventOrigin origin: EventOrigin.values()) {
for (final EventOrigin origin: EventOrigin.values()) {
checkInterruption();
if (ResultURLs.getStackSize(origin) > 1000) {
if (this.log.isFine()) log.logFine("Cleaning Loaded-URLs report stack, " + ResultURLs.getStackSize(origin) + " entries on stack " + origin.getCode());
if (this.log.isFine()) {
log.logFine("Cleaning Loaded-URLs report stack, " + ResultURLs.getStackSize(origin) + " entries on stack " + origin.getCode());
}
ResultURLs.clearStack(origin);
}
}
@ -1573,7 +1607,9 @@ public final class Switchboard extends serverSwitch {
// clean up news
checkInterruption();
try {
if (this.log.isFine()) log.logFine("Cleaning Incoming News, " + this.peers.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
if (this.log.isFine()) {
log.logFine("Cleaning Incoming News, " + this.peers.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
}
this.peers.newsPool.automaticProcess(peers);
} catch (final Exception e) {
Log.logException(e);
@ -1590,7 +1626,7 @@ public final class Switchboard extends serverSwitch {
final long deleteOldSeedsTime = getConfigLong("routing.deleteOldSeeds.time",7)*24*3600000;
Iterator<yacySeed> e = this.peers.seedsSortedDisconnected(true,yacySeed.LASTSEEN);
yacySeed seed = null;
final ArrayList<String> deleteQueue = new ArrayList<String>();
final List<String> deleteQueue = new ArrayList<String>();
checkInterruption();
// clean passive seeds
while (e.hasNext()) {
@ -1646,7 +1682,11 @@ public final class Switchboard extends serverSwitch {
profile.load(fileIn);
} catch(final IOException e) {
} finally {
if (fileIn != null) try { fileIn.close(); } catch (final Exception e) {}
if (fileIn != null) {
try {
fileIn.close();
} catch (final Exception e) {}
}
}
final String homepage = (String) profile.get("homepage");
if ((homepage != null) && (homepage.length() > 10)) {
@ -1660,8 +1700,9 @@ public final class Switchboard extends serverSwitch {
this.clusterhashes = this.peers.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
// check if we are reachable and try to map port again if not (e.g. when router rebooted)
if (getConfigBool(SwitchboardConstants.UPNP_ENABLED, false) && sb.peers.mySeed().isJunior())
if (getConfigBool(SwitchboardConstants.UPNP_ENABLED, false) && sb.peers.mySeed().isJunior()) {
UPnP.addPortMapping();
}
// after all clean up is done, check the resource usage
observer.resourceObserverJob();
@ -1720,7 +1761,9 @@ public final class Switchboard extends serverSwitch {
in.queueEntry.updateStatus(Response.QUEUE_STATE_PARSING);
// debug
if (log.isFinest()) log.logFinest("PARSE "+ in.queueEntry);
if (log.isFinest()) {
log.logFinest("PARSE "+ in.queueEntry);
}
Document[] documents = null;
try {
@ -1740,7 +1783,8 @@ public final class Switchboard extends serverSwitch {
Document[] documents = null;
final EventOrigin processCase = response.processCase(peers.mySeed().hash);
if (this.log.isFine()) log.logFine("processResourceStack processCase=" + processCase +
if (this.log.isFine()) {
log.logFine("processResourceStack processCase=" + processCase +
", depth=" + response.depth() +
", maxDepth=" + ((response.profile() == null) ? "null" : Integer.toString(response.profile().depth())) +
", must-match=" + ((response.profile() == null) ? "null" : response.profile().mustMatchPattern().toString()) +
@ -1748,6 +1792,7 @@ public final class Switchboard extends serverSwitch {
", initiatorHash=" + ((response.initiator() == null) ? "null" : UTF8.String(response.initiator())) +
//", responseHeader=" + ((entry.responseHeader() == null) ? "null" : entry.responseHeader().toString()) +
", url=" + response.url()); // DEBUG
}
// PARSE CONTENT
final long parsingStartTime = System.currentTimeMillis();
@ -1766,7 +1811,9 @@ public final class Switchboard extends serverSwitch {
try {
// parse the document
documents = TextParser.parseSource(response.url(), response.getMimeType(), response.getCharacterEncoding(), b);
if (documents == null) throw new Parser.Failure("Parser returned null.", response.url());
if (documents == null) {
throw new Parser.Failure("Parser returned null.", response.url());
}
} catch (final Parser.Failure e) {
this.log.logWarning("Unable to parse the resource '" + response.url() + "'. " + e.getMessage());
addURLtoErrorDB(response.url(), response.referrerHash(), response.initiator(), response.name(), e.getMessage());
@ -1816,9 +1863,11 @@ public final class Switchboard extends serverSwitch {
}
}
final long stackEndTime = System.currentTimeMillis();
if (log.isInfo()) log.logInfo("CRAWL: ADDED " + hl.size() + " LINKS FROM " + response.url().toNormalform(false, true) +
if (log.isInfo()) {
log.logInfo("CRAWL: ADDED " + hl.size() + " LINKS FROM " + response.url().toNormalform(false, true) +
", STACKING TIME = " + (stackEndTime-stackStartTime) +
", PARSING TIME = " + (parsingEndTime-parsingStartTime));
}
}
return documents;
}
@ -1826,15 +1875,19 @@ public final class Switchboard extends serverSwitch {
public indexingQueueEntry condenseDocument(final indexingQueueEntry in) {
in.queueEntry.updateStatus(Response.QUEUE_STATE_CONDENSING);
if (!in.queueEntry.profile().indexText() && !in.queueEntry.profile().indexMedia()) {
if (log.isInfo()) log.logInfo("Not Condensed Resource '" + in.queueEntry.url().toNormalform(false, true) + "': indexing not wanted by crawl profile");
if (log.isInfo()) {
log.logInfo("Not Condensed Resource '" + in.queueEntry.url().toNormalform(false, true) + "': indexing not wanted by crawl profile");
}
return new indexingQueueEntry(in.process, in.queueEntry, in.documents, null);
}
List<Document> doclist = new ArrayList<Document>();
// check which files may take part in the indexing process
for (Document document: in.documents) {
for (final Document document: in.documents) {
if (document.indexingDenied()) {
if (log.isInfo()) log.logInfo("Not Condensed Resource '" + in.queueEntry.url().toNormalform(false, true) + "': denied by document-attached noindexing rule");
if (log.isInfo()) {
log.logInfo("Not Condensed Resource '" + in.queueEntry.url().toNormalform(false, true) + "': denied by document-attached noindexing rule");
}
addURLtoErrorDB(in.queueEntry.url(), in.queueEntry.referrerHash(), in.queueEntry.initiator(), in.queueEntry.name(), "denied by document-attached noindexing rule");
continue;
}
@ -1843,8 +1896,10 @@ public final class Switchboard extends serverSwitch {
if (doclist.isEmpty()) return new indexingQueueEntry(in.process, in.queueEntry, in.documents, null);
in.documents = doclist.toArray(new Document[doclist.size()]);
Condenser[] condenser = new Condenser[in.documents.length];
if (this.log.isFine()) log.logFine("Condensing for '" + in.queueEntry.url().toNormalform(false, true) + "'");
final Condenser[] condenser = new Condenser[in.documents.length];
if (this.log.isFine()) {
log.logFine("Condensing for '" + in.queueEntry.url().toNormalform(false, true) + "'");
}
for (int i = 0; i < in.documents.length; i++) {
condenser[i] = new Condenser(in.documents[i], in.queueEntry.profile().indexText(), in.queueEntry.profile().indexMedia(), LibraryProvider.dymLib);
@ -1875,8 +1930,10 @@ public final class Switchboard extends serverSwitch {
public void storeDocumentIndex(final indexingQueueEntry in) {
in.queueEntry.updateStatus(Response.QUEUE_STATE_INDEXSTORAGE);
// the condenser may be null in case that an indexing is not wanted (there may be a no-indexing flag in the file)
if (in.condenser != null) for (int i = 0; i < in.documents.length; i++) {
storeDocumentIndex(in.process, in.queueEntry, in.documents[i], in.condenser[i], null, "crawler/indexing queue");
if (in.condenser != null) {
for (int i = 0; i < in.documents.length; i++) {
storeDocumentIndex(in.process, in.queueEntry, in.documents[i], in.condenser[i], null, "crawler/indexing queue");
}
}
in.queueEntry.updateStatus(Response.QUEUE_STATE_FINISHED);
}
@ -1887,7 +1944,9 @@ public final class Switchboard extends serverSwitch {
final String dc_title = document.dc_title();
final DigestURI referrerURL = queueEntry.referrerURL();
EventOrigin processCase = queueEntry.processCase(peers.mySeed().hash);
if (process == Segments.Process.SURROGATES) processCase = EventOrigin.SURROGATES;
if (process == Segments.Process.SURROGATES) {
processCase = EventOrigin.SURROGATES;
}
if (condenser == null || document.indexingDenied()) {
//if (this.log.isInfo()) log.logInfo("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': denied by rule in document, process case=" + processCase);
@ -1962,7 +2021,9 @@ public final class Switchboard extends serverSwitch {
if ((processCase == EventOrigin.GLOBAL_CRAWLING) && (queueEntry.initiator() != null)) {
final yacySeed initiatorPeer = peers.get(UTF8.String(queueEntry.initiator()));
if (initiatorPeer != null) {
if (clusterhashes != null) initiatorPeer.setAlternativeAddress(clusterhashes.get(queueEntry.initiator()));
if (clusterhashes != null) {
initiatorPeer.setAlternativeAddress(clusterhashes.get(queueEntry.initiator()));
}
// start a thread for receipt sending to avoid a blocking here
new Thread(new receiptSending(initiatorPeer, newEntry), "sending receipt to " + UTF8.String(queueEntry.initiator())).start();
}
@ -1972,25 +2033,34 @@ public final class Switchboard extends serverSwitch {
public final void addAllToIndex(final DigestURI url, final Map<MultiProtocolURI, String> links, final SearchEvent searchEvent, final String heuristicName) {
// add the landing page to the index. should not load that again since it should be in the cache
if (url != null) try {
this.addToIndex(url, searchEvent, heuristicName);
} catch (IOException e) {} catch (Parser.Failure e) {}
if (url != null) {
try {
this.addToIndex(url, searchEvent, heuristicName);
} catch (IOException e) {
} catch (Parser.Failure e) {
}
}
// check if some of the links match with the query
Map<MultiProtocolURI, String> matcher = searchEvent.getQuery().separateMatches(links);
final Map<MultiProtocolURI, String> matcher = searchEvent.getQuery().separateMatches(links);
// take the matcher and load them all
for (final Map.Entry<MultiProtocolURI, String> entry: matcher.entrySet()) {
try {
this.addToIndex(new DigestURI(entry.getKey(), (byte[]) null), searchEvent, heuristicName);
} catch (IOException e) {} catch (Parser.Failure e) {}
} catch (IOException e) {
} catch (Parser.Failure e) {
}
}
// take then the no-matcher and load them also
for (final Map.Entry<MultiProtocolURI, String> entry: links.entrySet()) {
try {
this.addToIndex(new DigestURI(entry.getKey(), (byte[]) null), searchEvent, heuristicName);
} catch (IOException e) {} catch (Parser.Failure e) {}
} catch (IOException e) {
} catch (Parser.Failure e) {
}
}
}
@ -2004,7 +2074,9 @@ public final class Switchboard extends serverSwitch {
*/
public void addToIndex(final DigestURI url, final SearchEvent searchEvent, final String heuristicName) throws IOException, Parser.Failure {
final Segments.Process process = Segments.Process.LOCALCRAWLING;
if (searchEvent != null) searchEvent.addHeuristic(url.hash(), heuristicName, true);
if (searchEvent != null) {
searchEvent.addHeuristic(url.hash(), heuristicName, true);
}
if (indexSegments.segment(process).urlMetadata.exists(url.hash())) return; // don't do double-work
final Request request = loader.request(url, true, true);
final CrawlProfile profile = sb.crawler.getActive(request.profileHandle().getBytes());
@ -2018,17 +2090,27 @@ public final class Switchboard extends serverSwitch {
public void run() {
try {
final Response response = loader.load(request, CacheStrategy.IFFRESH, Long.MAX_VALUE, true);
if (response == null) throw new IOException("response == null");
if (response.getContent() == null) throw new IOException("content == null");
if (response.getResponseHeader() == null) throw new IOException("header == null");
if (response == null) {
throw new IOException("response == null");
}
if (response.getContent() == null) {
throw new IOException("content == null");
}
if (response.getResponseHeader() == null) {
throw new IOException("header == null");
}
final Document[] documents = response.parse();
if (documents != null) for (final Document document: documents) {
if (document.indexingDenied()) throw new Parser.Failure("indexing is denied", url);
final Condenser condenser = new Condenser(document, true, true, LibraryProvider.dymLib);
ResultImages.registerImages(url, document, true);
webStructure.generateCitationReference(url, document, condenser, response.lastModified());
storeDocumentIndex(process, response, document, condenser, searchEvent, "heuristic:" + heuristicName);
log.logInfo("addToIndex fill of url " + url.toNormalform(true, true) + " finished");
if (documents != null) {
for (final Document document: documents) {
if (document.indexingDenied()) {
throw new Parser.Failure("indexing is denied", url);
}
final Condenser condenser = new Condenser(document, true, true, LibraryProvider.dymLib);
ResultImages.registerImages(url, document, true);
webStructure.generateCitationReference(url, document, condenser, response.lastModified());
storeDocumentIndex(process, response, document, condenser, searchEvent, "heuristic:" + heuristicName);
log.logInfo("addToIndex fill of url " + url.toNormalform(true, true) + " finished");
}
}
} catch (IOException e) {
log.logWarning("addToIndex: failed loading " + url.toNormalform(false, false) + ": " + e.getMessage());
@ -2048,13 +2130,13 @@ public final class Switchboard extends serverSwitch {
this.reference = reference;
}
public void run() {
long t = System.currentTimeMillis();
Map<String, String> response = yacyClient.crawlReceipt(peers.mySeed(), initiatorPeer, "crawl", "fill", "indexed", reference, "");
final long t = System.currentTimeMillis();
final Map<String, String> response = yacyClient.crawlReceipt(peers.mySeed(), initiatorPeer, "crawl", "fill", "indexed", reference, "");
if (response == null) {
log.logInfo("Sending crawl receipt for '" + reference.metadata().url().toNormalform(false, true) + "' to " + initiatorPeer.getName() + " FAILED, send time = " + (System.currentTimeMillis() - t));
return;
}
String delay = response.get("delay");
final String delay = response.get("delay");
log.logInfo("Sending crawl receipt for '" + reference.metadata().url().toNormalform(false, true) + "' to " + initiatorPeer.getName() + " success, delay = " + delay + ", send time = " + (System.currentTimeMillis() - t));
}
}
@ -2105,8 +2187,9 @@ public final class Switchboard extends serverSwitch {
return true;
case 4: // hard-authenticated, all ok
return true;
default:
return false;
}
return false;
}
public void setPerformance(final int wantedPPM) {
@ -2115,8 +2198,12 @@ public final class Switchboard extends serverSwitch {
// wantedPPM <= 10: low performance
// 10 < wantedPPM < 1000: custom performance
// 1000 <= wantedPPM : maximum performance
if (wPPM <= 10) wPPM = 10;
if (wPPM >= 30000) wPPM = 30000;
if (wPPM <= 10) {
wPPM = 10;
}
if (wPPM >= 30000) {
wPPM = 30000;
}
final int newBusySleep = 60000 / wPPM; // for wantedPPM = 10: 6000; for wantedPPM = 1000: 60
BusyThread thread;
@ -2188,7 +2275,9 @@ public final class Switchboard extends serverSwitch {
public boolean dhtTransferJob(final String segment) {
final String rejectReason = dhtShallTransfer(segment);
if (rejectReason != null) {
if (this.log.isFine()) log.logFine(rejectReason);
if (this.log.isFine()) {
log.logFine(rejectReason);
}
return false;
}
boolean hasDoneSomething = false;
@ -2211,7 +2300,9 @@ public final class Switchboard extends serverSwitch {
startHash = PeerSelection.selectTransferStart();
assert startHash != null;
limitHash = PeerSelection.limitOver(this.peers, startHash);
if (limitHash != null) break;
if (limitHash != null) {
break;
}
}
if (limitHash == null || startHash == null) {
log.logInfo("dhtTransferJob: approaching full DHT dispersion.");
@ -2274,7 +2365,9 @@ public final class Switchboard extends serverSwitch {
@Override
public void run() {
String r = host;
if (r.indexOf("//") < 0) r = "http://" + r;
if (r.indexOf("//") < 0) {
r = "http://" + r;
}
// get the links for a specific site
DigestURI url;
@ -2294,7 +2387,9 @@ public final class Switchboard extends serverSwitch {
}
final Iterator<MultiProtocolURI> i = links.keySet().iterator();
while (i.hasNext()) {
if (!i.next().getHost().endsWith(host)) i.remove();
if (!i.next().getHost().endsWith(host)) {
i.remove();
}
}
// add all pages to the index
@ -2311,7 +2406,7 @@ public final class Switchboard extends serverSwitch {
int meta = query.indexOf("heuristic:");
if (meta >= 0) {
final int q = query.indexOf(' ', meta);
if (q >= 0) query = query.substring(0, meta) + query.substring(q + 1); else query = query.substring(0, meta);
query = (q >= 0) ? query.substring(0, meta) + query.substring(q + 1) : query.substring(0, meta);
}
final String urlString = "http://www.scroogle.org/cgi-bin/nbbw.cgi?Gw=" + query.trim().replaceAll(" ", "+") + "&n=2";
final DigestURI url;
@ -2329,9 +2424,11 @@ public final class Switchboard extends serverSwitch {
Log.logException(e);
return;
}
Iterator<MultiProtocolURI> i = links.keySet().iterator();
final Iterator<MultiProtocolURI> i = links.keySet().iterator();
while (i.hasNext()) {
if (i.next().toNormalform(false, false).indexOf("scroogle") >= 0) i.remove();
if (i.next().toNormalform(false, false).indexOf("scroogle") >= 0) {
i.remove();
}
}
log.logInfo("Heuristic: adding " + links.size() + " links from scroogle");
// add all pages to the index
@ -2366,8 +2463,8 @@ public final class Switchboard extends serverSwitch {
// if we have an url then try to load the rss
RSSReader rss = null;
try {
Response response = sb.loader.load(sb.loader.request(url, true, false), CrawlProfile.CacheStrategy.NOCACHE, Long.MAX_VALUE, true);
byte[] resource = response == null ? null : response.getContent();
final Response response = sb.loader.load(sb.loader.request(url, true, false), CrawlProfile.CacheStrategy.NOCACHE, Long.MAX_VALUE, true);
byte[] resource = (response == null) ? null : response.getContent();
//System.out.println("BLEKKO: " + UTF8.String(resource));
rss = resource == null ? null : RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, resource);
} catch (IOException e) {
@ -2380,7 +2477,7 @@ public final class Switchboard extends serverSwitch {
final Map<MultiProtocolURI, String> links = new TreeMap<MultiProtocolURI, String>();
MultiProtocolURI uri;
for (RSSMessage message: rss.getFeed()) try {
for (final RSSMessage message: rss.getFeed()) try {
uri = new MultiProtocolURI(message.getLink());
links.put(uri, message.getTitle());
} catch (MalformedURLException e) {
@ -2461,9 +2558,13 @@ public final class Switchboard extends serverSwitch {
// - use the superseed to further fill up the seedDB
int ssc = 0, c = 0;
while (true) {
if (Thread.currentThread().isInterrupted()) break;
if (Thread.currentThread().isInterrupted()) {
break;
}
seedListFileURL = sb.getConfig("network.unit.bootstrap.seedlist" + c, "");
if (seedListFileURL.length() == 0) break;
if (seedListFileURL.length() == 0) {
break;
}
c++;
if (
seedListFileURL.startsWith("http://") ||
@ -2497,9 +2598,11 @@ public final class Switchboard extends serverSwitch {
ys = yacySeed.genRemoteSeed(enu.next(), null, false);
if ((ys != null) &&
(!peers.mySeedIsDefined() || !peers.mySeed().hash.equals(ys.hash))) {
long lastseen = Math.abs((System.currentTimeMillis() - ys.getLastSeenUTC()) / 1000 / 60);
final long lastseen = Math.abs((System.currentTimeMillis() - ys.getLastSeenUTC()) / 1000 / 60);
if (lastseen < 240) {
if (peers.peerActions.connectPeer(ys, false)) lc++;
if (peers.peerActions.connectPeer(ys, false)) {
lc++;
}
}
}
} catch (IOException e) {
@ -2563,12 +2666,17 @@ public final class Switchboard extends serverSwitch {
public void checkInterruption() throws InterruptedException {
final Thread curThread = Thread.currentThread();
if ((curThread instanceof WorkflowThread) && ((WorkflowThread)curThread).shutdownInProgress()) throw new InterruptedException("Shutdown in progress ...");
else if (this.terminate || curThread.isInterrupted()) throw new InterruptedException("Shutdown in progress ...");
if ((curThread instanceof WorkflowThread) && ((WorkflowThread)curThread).shutdownInProgress()) {
throw new InterruptedException("Shutdown in progress ...");
} else if (this.terminate || curThread.isInterrupted()) {
throw new InterruptedException("Shutdown in progress ...");
}
}
public void terminate(final long delay, final String reason) {
if (delay <= 0) throw new IllegalArgumentException("The shutdown delay must be greater than 0.");
if (delay <= 0) {
throw new IllegalArgumentException("The shutdown delay must be greater than 0.");
}
log.logInfo("caught delayed terminate request: " + reason);
(new delayedShutdown(this, delay, reason)).start();
}
@ -2607,10 +2715,8 @@ public final class Switchboard extends serverSwitch {
client.setHeader(reqHeader.entrySet());
try {
// sending request
// final Map<String, String> result = FileUtils.table(Client.wget(url.toString(), reqHeader, 10000));
final Map<String, String> result = FileUtils.table(client.GETbytes(url.toString()));
if (result == null) return new HashMap<String, String>();
return result;
return (result == null) ? new HashMap<String, String>() : result;
} catch (final Exception e) {
Log.logException(e);
return new HashMap<String, String>();

2
source/net/yacy/ai/greedy/Asset.java
Unescape View File

@ -33,7 +33,7 @@ public class Asset<
private final SpecificModel model;
private final SpecificFinding finding;
public Asset(SpecificModel model, SpecificFinding finding) {
public Asset(final SpecificModel model, final SpecificFinding finding) {
this.model = model;
this.finding = finding;
}

Write Preview
Loading…
Cancel
Save