boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) minor changes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7570 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 14 years ago
parent 8b94e33a33
commit 3e03963b1c
3 changed files with 116 additions and 108 deletions
Show Stats Download Patch File Download Diff File

8
htroot/ViewLog_p.java
Unescape View File

@ -64,11 +64,7 @@ public class ViewLog_p {
json = post.containsKey("json");
if(post.containsKey("lines")){
try {
lines = Integer.parseInt(post.get("lines"));
} catch (NumberFormatException e) {
Log.logException(e);
}
lines = post.getInt("lines", lines);
}
if(post.containsKey("filter")){
@ -76,7 +72,6 @@ public class ViewLog_p {
}
}
final Logger logger = Logger.getLogger("");
final Handler[] handlers = logger.getHandlers();
boolean displaySubmenu = false;
@ -105,7 +100,6 @@ public class ViewLog_p {
Log.logException(e);
}
int level = 0;
int lc = 0;
for (final String logLine : log) {

78
htroot/yacysearch.java
Unescape View File

@ -76,6 +76,7 @@ import de.anomic.server.servletProperties;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.graphics.ProfilingGraph;
import de.anomic.yacy.yacyChannel;
import java.util.Map;
public class yacysearch {
@ -87,12 +88,14 @@ public class yacysearch {
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final boolean localhostAccess = sb.accessFromLocalhost(header);
String promoteSearchPageGreeting = env.getConfig(SwitchboardConstants.GREETING, "");
if (env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
final String promoteSearchPageGreeting =
(env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ?
env.getConfig("network.unit.description", "") :
env.getConfig(SwitchboardConstants.GREETING, "");
final String client = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
// get query
String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim();
final String originalquerystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim();
String querystring = originalquerystring.replace('+', ' ').replace('*', ' ').trim();
CrawlProfile.CacheStrategy snippetFetchStrategy = (post == null) ? null : CrawlProfile.CacheStrategy.parse(post.get("verify", "cacheonly"));
final servletProperties prop = new servletProperties();
@ -101,7 +104,7 @@ public class yacysearch {
// get segment
Segment indexSegment = null;
if (post != null && post.containsKey("segment")) {
String segmentName = post.get("segment");
final String segmentName = post.get("segment");
if (sb.indexSegments.segmentExist(segmentName)) {
indexSegment = sb.indexSegments.segment(segmentName);
}
@ -166,19 +169,18 @@ public class yacysearch {
}
// collect search attributes
boolean newsearch = post.hasValue("query") && post.hasValue("former") && !post.get("query","").equalsIgnoreCase(post.get("former","")); //new search term
final boolean newsearch =post.hasValue("query") && post.hasValue("former") && !post.get("query","").equalsIgnoreCase(post.get("former","")); //new search term
int itemsPerPage = Math.min((authenticated) ? (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 100 : 1000) : (snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ? 20 : 500), post.getInt("maximumRecords", post.getInt("count", 10))); // SRU syntax with old property as alternative
int offset = (newsearch) ? 0 : post.getInt("startRecord", post.getInt("offset", 0));
int newcount;
final int newcount;
if ( authenticated && (newcount = post.getInt("count", 0)) > 0 ) sb.setConfig(SwitchboardConstants.SEARCH_ITEMS, newcount); // set new default maximumRecords if search with "more options"
boolean global = post.get("resource", "local").equals("global") && sb.peers.sizeConnected() > 0;
final boolean indexof = (post != null && post.get("indexof","").equals("on"));
String urlmask = null;
String originalUrlMask = null;
final String originalUrlMask;
if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) { // option search all
originalUrlMask = ".*";
} else if (!newsearch && post.containsKey("urlmaskfilter")) {
@ -188,7 +190,9 @@ public class yacysearch {
}
String prefermask = (post == null) ? "" : post.get("prefermaskfilter", "");
if (prefermask.length() > 0 && prefermask.indexOf(".*") < 0) prefermask = ".*" + prefermask + ".*";
if (prefermask.length() > 0 && prefermask.indexOf(".*") < 0) {
prefermask = ".*" + prefermask + ".*";
}
Bitfield constraint = (post != null && post.containsKey("constraint") && post.get("constraint", "").length() > 0) ? new Bitfield(4, post.get("constraint", "______")) : null;
if (indexof) {
@ -200,12 +204,10 @@ public class yacysearch {
final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true) ||
sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, true);
global = global && indexReceiveGranted; // if the user does not want indexes from remote peers, it cannot be a global search
//final boolean offline = yacyCore.seedDB.mySeed().isVirgin();
final boolean clustersearch = sb.isRobinsonMode() &&
(sb.getConfig("cluster.mode", "").equals("privatecluster") ||
sb.getConfig("cluster.mode", "").equals("publiccluster"));
//if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; }
if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes
// increase search statistic counter
@ -269,6 +271,7 @@ public class yacysearch {
}
if ((!block) && (post == null || post.get("cat", "href").equals("href"))) {
String urlmask = null;
// check available memory and clean up if necessary
if (!MemoryControl.request(8000000L, false)) {
@ -289,12 +292,14 @@ public class yacysearch {
int lrp = querystring.indexOf("/language/");
String lr = "";
if (lrp >= 0) {
if (querystring.length() >= (lrp + 11))
if (querystring.length() >= (lrp + 11)) {
lr = querystring.substring(lrp + 9, lrp + 11);
}
querystring = querystring.replace("/language/" + lr, "");
lr = lr.toLowerCase();
}
int inurl = querystring.indexOf("inurl:");
final int inurl = querystring.indexOf("inurl:");
if (inurl >= 0) {
int ftb = querystring.indexOf(' ', inurl);
if (ftb == -1) ftb = querystring.length();
@ -302,7 +307,7 @@ public class yacysearch {
querystring = querystring.replace("inurl:" + urlstr, "");
if(urlstr.length() > 0) urlmask = ".*" + urlstr + ".*";
}
int filetype = querystring.indexOf("filetype:");
final int filetype = querystring.indexOf("filetype:");
if (filetype >= 0) {
int ftb = querystring.indexOf(' ', filetype);
if (ftb == -1) ftb = querystring.length();
@ -338,24 +343,21 @@ public class yacysearch {
sitehash = DigestURI.domhash(sitehost);
}
int heuristicScroogle = querystring.indexOf("heuristic:scroogle");
final int heuristicScroogle = querystring.indexOf("heuristic:scroogle");
if (heuristicScroogle >= 0) {
querystring = querystring.replace("heuristic:scroogle", "");
}
int heuristicBlekko = querystring.indexOf("heuristic:blekko");
final int heuristicBlekko = querystring.indexOf("heuristic:blekko");
if (heuristicBlekko >= 0) {
querystring = querystring.replace("heuristic:blekko", "");
}
int authori = querystring.indexOf("author:");
final int authori = querystring.indexOf("author:");
String authorhash = null;
if (authori >= 0) {
// check if the author was given with single quotes or without
boolean quotes = false;
if (querystring.charAt(authori + 7) == (char) 39) {
quotes = true;
}
final boolean quotes = (querystring.charAt(authori + 7) == (char) 39);
String author;
if (quotes) {
int ftb = querystring.indexOf((char) 39, authori + 8);
@ -370,7 +372,7 @@ public class yacysearch {
}
authorhash = UTF8.String(Word.word2hash(author));
}
int tld = querystring.indexOf("tld:");
final int tld = querystring.indexOf("tld:");
if (tld >= 0) {
int ftb = querystring.indexOf(' ', tld);
if (ftb == -1) ftb = querystring.length();
@ -401,7 +403,7 @@ public class yacysearch {
}
// navigation
String navigation = (post == null) ? "" : post.get("nav", "");
final String navigation = (post == null) ? "" : post.get("nav", "");
// the query
final TreeSet<String>[] query = QueryParams.cleanQuery(querystring.trim()); // converts also umlaute
@ -427,7 +429,7 @@ public class yacysearch {
// make new news message with negative voting
if (!sb.isRobinsonMode()) {
final HashMap<String, String> map = new HashMap<String, String>();
final Map<String, String> map = new HashMap<String, String>();
map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
@ -455,7 +457,7 @@ public class yacysearch {
}
if (documents != null) {
// create a news message
final HashMap<String, String> map = new HashMap<String, String>();
final Map<String, String> map = new HashMap<String, String>();
map.put("url", metadata.url().toNormalform(false, true).replace(',', '|'));
map.put("title", metadata.dc_title().replace(',', ' '));
map.put("description", documents[0].dc_title().replace(',', ' '));
@ -468,7 +470,6 @@ public class yacysearch {
}
// prepare search properties
//final boolean yacyonline = ((sb.webIndex.seedDB != null) && (sb.webIndex.seedDB.mySeed() != null) && (sb.webIndex.seedDB.mySeed().getPublicAddress() != null));
final boolean globalsearch = (global) && indexReceiveGranted; /* && (yacyonline)*/
// do the search
@ -527,7 +528,9 @@ public class yacysearch {
sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)),
(int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_ROBINSON, 0),
(int) sb.getConfigLong(SwitchboardConstants.DHT_BURST_MULTIWORD, 0));
try {Thread.sleep(global ? 100 : 10);} catch (InterruptedException e1) {} // wait a little time to get first results in the search
try {
Thread.sleep(global ? 100 : 10);
} catch (InterruptedException e1) {} // wait a little time to get first results in the search
if (offset == 0) {
if (sitehost != null && sb.getConfigBool("heuristic.site", false) && authenticated) sb.heuristicSite(theSearch, sitehost);
@ -535,13 +538,6 @@ public class yacysearch {
if ((heuristicBlekko >= 0 || sb.getConfigBool("heuristic.blekko", false)) && authenticated) sb.heuristicRSS("http://blekko.com/ws/$+/rss", theSearch, "blekko");
}
// generate result object
//serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER ORDERING OF SEARCH RESULTS: " + (System.currentTimeMillis() - timestamp) + " ms");
//serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER RESULT PREPARATION: " + (System.currentTimeMillis() - timestamp) + " ms");
// calc some more cross-reference
//serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER XREF PREPARATION: " + (System.currentTimeMillis() - timestamp) + " ms");
// log
Log.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
"local-unfiltered(" + theSearch.getRankingResult().getLocalIndexCount() + "), " +
@ -565,8 +561,8 @@ public class yacysearch {
}
prop.put("meanCount", meanMax);
if (meanMax > 0) {
DidYouMean didYouMean = new DidYouMean(indexSegment.termIndex(), querystring);
Iterator<String> meanIt = didYouMean.getSuggestions(100, 5).iterator();
final DidYouMean didYouMean = new DidYouMean(indexSegment.termIndex(), querystring);
final Iterator<String> meanIt = didYouMean.getSuggestions(100, 5).iterator();
int meanCount = 0;
String suggestion;
while(meanCount<meanMax && meanIt.hasNext()) {
@ -606,7 +602,9 @@ public class yacysearch {
try {
synchronized (trackerHandles) {
trackerHandles.add(theQuery.time);
while (trackerHandles.size() > 600) if (!trackerHandles.remove(trackerHandles.first())) break;
while (trackerHandles.size() > 600) {
if (!trackerHandles.remove(trackerHandles.first())) break;
}
}
sb.localSearchTracker.put(client, trackerHandles);
if (sb.localSearchTracker.size() > 1000) sb.localSearchTracker.remove(sb.localSearchTracker.keys().nextElement());
@ -614,7 +612,7 @@ public class yacysearch {
Log.logException(e);
}
int indexcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() + theSearch.getRankingResult().getRemoteIndexCount();
final int indexcount = theSearch.getRankingResult().getLocalIndexCount() - theSearch.getRankingResult().getMissCount() + theSearch.getRankingResult().getRemoteIndexCount();
prop.put("num-results_offset", offset);
prop.put("num-results_itemscount", Formatter.number(0, true));
prop.put("num-results_itemsPerPage", itemsPerPage);
@ -662,7 +660,7 @@ public class yacysearch {
resnav.append(QueryParams.navurl("html", thispage + 1, theQuery, null, originalUrlMask, navigation));
resnav.append("\"><img src=\"env/grafics/navdr.gif\" alt=\"arrowright\" width=\"16\" height=\"16\" /></a>");
}
String resnavs = resnav.toString();
final String resnavs = resnav.toString();
prop.put("num-results_resnav", resnavs);
prop.put("pageNavBottom", (indexcount - offset > 6) ? 1 : 0); // if there are more results than may fit on the page we add a navigation at the bottom
prop.put("pageNavBottom_resnav", resnavs);
@ -685,7 +683,7 @@ public class yacysearch {
}
if (prop == null || prop.isEmpty()) {
if (post == null || post.get("query", post.get("search", "")).length() < 3) {
if (post.get("query", post.get("search", "")).length() < 3) {
prop.put("num-results", "2"); // no results - at least 3 chars
} else {
prop.put("num-results", "1"); // no results

66
htroot/yacysearchitem.java
Unescape View File

@ -26,7 +26,7 @@
import java.net.MalformedURLException;
import java.util.List;
import java.util.TreeSet;
import java.util.Set;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.UTF8;
@ -57,9 +57,12 @@ import de.anomic.yacy.graphics.ProfilingGraph;
public class yacysearchitem {
private static final String SHORTEN_SUFFIX = "...";
private static final int SHORTEN_SUFFIX_LENGTH = SHORTEN_SUFFIX.length();
private static final int MAX_NAME_LENGTH = 60;
private static final int MAX_URL_LENGTH = 120;
private static boolean col = true;
private static final int namelength = 60;
private static final int urllength = 120;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
@ -69,6 +72,9 @@ public class yacysearchitem {
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int item = post.getInt("item", -1);
final boolean auth = (header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")).equals("localhost") || sb.verifyAuthentication(header, true);
final String path = header.get(HeaderFramework.CONNECTION_PROP_PATH);
final boolean isHtml = path.endsWith(".html");
final boolean isJson = path.endsWith(".json");
// default settings for blank item
prop.put("content", "0");
@ -76,9 +82,6 @@ public class yacysearchitem {
prop.put("references", "0");
prop.put("rssreferences", "0");
prop.put("dynamic", "0");
String p = header.get(HeaderFramework.CONNECTION_PROP_PATH);
boolean isHtml = p.endsWith(".html");
boolean isJson = p.endsWith(".json");
// find search event
final SearchEvent theSearch = SearchEventCache.getEvent(eventID);
@ -100,7 +103,7 @@ public class yacysearchitem {
prop.put("remoteIndexCount", Formatter.number(theSearch.getRankingResult().getRemoteIndexCount(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.getRankingResult().getRemotePeerCount(), true));
String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET, "_self");
final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET, "_self");
if (theQuery.contentdom == ContentDomain.TEXT) {
// text search
@ -108,7 +111,7 @@ public class yacysearchitem {
final ResultEntry result = theSearch.oneResult(item, theQuery.isLocal() ? 1000 : 5000);
if (result == null) return prop; // no content
DigestURI resultURL = result.url();
final DigestURI resultURL = result.url();
final int port = resultURL.getPort();
DigestURI faviconURL = null;
if ((isHtml || isJson) && !sb.isIntranetMode() && !resultURL.isLocal()) try {
@ -142,7 +145,7 @@ public class yacysearchitem {
prop.put("content_showMetadata_urlhash", resulthashString);
prop.put("content_showParser_urlhash", resulthashString);
prop.put("content_urlhexhash", yacySeed.b64Hash2hexHash(resulthashString));
prop.putHTML("content_urlname", nxTools.shortenURLString(result.urlname(), urllength));
prop.putHTML("content_urlname", nxTools.shortenURLString(result.urlname(), MAX_URL_LENGTH));
prop.put("content_showDate_date", GenericFormatter.RFC1123_SHORT_FORMATTER.format(result.modified()));
prop.put("content_date822", HeaderFramework.formatRFC1123(result.modified()));
//prop.put("content_ybr", RankingProcess.ybr(result.hash()));
@ -156,11 +159,14 @@ public class yacysearchitem {
prop.putHTML("content_publisher", result.publisher());
prop.putHTML("content_creator", result.creator());// author
prop.putHTML("content_subject", result.subject());
final TreeSet<String>[] query = theQuery.queryWords();
String s = ""; for (String t: query[0]) s += "+" + t;
if (s.length() > 0) s = s.substring(1);
prop.putHTML("content_words", s);
prop.putHTML("content_showParser_words", s);
final Set<String>[] query = theQuery.queryWords();
final StringBuilder s = new StringBuilder();
for (final String t: query[0]) {
s.append("+").append(t);
}
final String words = (s.length() > 0) ? s.substring(1) : "";
prop.putHTML("content_words", words);
prop.putHTML("content_showParser_words", words);
prop.putHTML("content_former", theQuery.queryString);
prop.putHTML("content_showPictures_former", theQuery.queryString);
final TextSnippet snippet = result.textSnippet();
@ -180,7 +186,7 @@ public class yacysearchitem {
prop.put("content_heuristic_name", heuristic.heuristicName);
}
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.searchEvent(theQuery.id(true), SearchEvent.Type.FINALIZATION, "" + item, 0, 0), false);
String ext = resultURL.getFileExtension().toLowerCase();
final String ext = resultURL.getFileExtension().toLowerCase();
if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) {
String license = sb.licensedURLs.aquireLicense(resultURL);
prop.put("content_code", license);
@ -191,7 +197,6 @@ public class yacysearchitem {
return prop;
}
if (theQuery.contentdom == ContentDomain.IMAGE) {
// image search; shows thumbnails
@ -200,12 +205,12 @@ public class yacysearchitem {
if (ms == null) {
prop.put("content_item", "0");
} else {
String license = sb.licensedURLs.aquireLicense(ms.href);
final String license = sb.licensedURLs.aquireLicense(ms.href);
sb.loader.loadIfNotExistBackground(ms.href.toNormalform(true, false), 1024 * 1024 * 10);
prop.putHTML("content_item_hrefCache", (auth) ? "/ViewImage.png?url=" + ms.href.toNormalform(true, false) : ms.href.toNormalform(true, false));
prop.putHTML("content_item_href", ms.href.toNormalform(true, false));
prop.put("content_item_code", license);
prop.putHTML("content_item_name", shorten(ms.name, namelength));
prop.putHTML("content_item_name", shorten(ms.name, MAX_NAME_LENGTH));
prop.put("content_item_mimetype", ms.mime);
prop.put("content_item_fileSize", ms.fileSize);
prop.put("content_item_width", ms.width);
@ -238,9 +243,9 @@ public class yacysearchitem {
int c = 0;
for (final MediaSnippet ms : media) {
prop.putHTML("content_items_" + c + "_href", ms.href.toNormalform(true, false));
prop.putHTML("content_items_" + c + "_hrefshort", nxTools.shortenURLString(ms.href.toNormalform(true, false), urllength));
prop.putHTML("content_items_" + c + "_hrefshort", nxTools.shortenURLString(ms.href.toNormalform(true, false), MAX_URL_LENGTH));
prop.putHTML("content_items_" + c + "_target", target);
prop.putHTML("content_items_" + c + "_name", shorten(ms.name, namelength));
prop.putHTML("content_items_" + c + "_name", shorten(ms.name, MAX_NAME_LENGTH));
prop.put("content_items_" + c + "_col", (col) ? "0" : "1");
c++;
col = !col;
@ -257,13 +262,24 @@ public class yacysearchitem {
}
private static String shorten(final String s, final int length) {
if (s.length() <= length) return s;
final String ret;
if (s.length() <= length) {
ret = s;
} else {
final int p = s.lastIndexOf('.');
if (p < 0) return s.substring(0, length - 3) + "...";
if (p < 0) {
ret = s.substring(0, length - SHORTEN_SUFFIX_LENGTH) + SHORTEN_SUFFIX;
} else {
assert p >= 0;
String ext = s.substring(p + 1);
if (ext.length() > 4) return s.substring(0, length / 2 - 2) + "..." + s.substring(s.length() - (length / 2 - 2));
return s.substring(0, length - ext.length() - 3) + "..." + ext;
final String ext = s.substring(p + 1);
if (ext.length() > 4) {
ret = s.substring(0, length / 2 - 2) + SHORTEN_SUFFIX + s.substring(s.length() - (length / 2 - 2));
} else {
ret = s.substring(0, length - ext.length() - SHORTEN_SUFFIX_LENGTH) + SHORTEN_SUFFIX + ext;
}
}
}
return ret;
}
private static String sizename(int size) {

Write Preview
Loading…
Cancel
Save