boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

prevent too high search request frequency submitted from the same peer

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4813 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 1127d62b64
commit 3aa69dab94
4 changed files with 99 additions and 26 deletions
Show Stats Download Patch File Download Diff File

32
htroot/yacy/search.java
Unescape View File

@ -121,6 +121,29 @@ public final class search {
return prop;
}
// check the search tracker
TreeSet<Long> trackerHandles = sb.remoteSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 6000)).size() > 1) try {
Thread.sleep(3000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 60000)).size() > 12) try {
Thread.sleep(10000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
Thread.sleep(30000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (block) {
prop.put("links", "");
prop.put("linkcount", "0");
prop.put("references", "");
return prop;
}
// tell all threads to do nothing for a specific time
sb.intermissionAllThreads(3000);
@ -291,10 +314,11 @@ public final class search {
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime();
sb.remoteSearches.add(theQuery);
TreeSet<Long> handles = sb.remoteSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(theQuery.handle);
sb.remoteSearchTracker.put(client, handles);
// update the search tracker
trackerHandles.add(theQuery.handle);
sb.remoteSearchTracker.put(client, trackerHandles);
// log
yacyCore.log.logInfo("EXIT HASH SEARCH: " +

31
htroot/yacy/ui/result.java
Unescape View File

@ -58,7 +58,8 @@ public class result {
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
if (promoteSearchPageGreeting.length() == 0) promoteSearchPageGreeting = "P2P WEB SEARCH";
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
// get query
String querystring = (post == null) ? "" : post.get("search", "").trim();
final serverObjects prop = new serverObjects();
@ -143,7 +144,24 @@ public class result {
// patch until better search profiles are available
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
if (post.get("cat", "href").equals("href")) {
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 3000)).size() > 1) try {
Thread.sleep(3000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 60000)).size() > 12) try {
Thread.sleep(10000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
Thread.sleep(30000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if ((!block) && (post.get("cat", "href").equals("href"))) {
final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
@ -166,7 +184,6 @@ public class result {
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
TreeSet<String> queryHashes = indexWord.words2hashes(query[0]);
plasmaSearchQuery theQuery = new plasmaSearchQuery(
querystring,
@ -226,10 +243,10 @@ public class result {
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
sb.localSearches.add(theQuery);
TreeSet<Long> handles = sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(theQuery.handle);
sb.localSearchTracker.put(client, handles);
// update the search tracker
trackerHandles.add(theQuery.handle);
sb.localSearchTracker.put(client, trackerHandles);
int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
prop.put("num-results_offset", offset);

31
htroot/yacy/user/ysearch.java
Unescape View File

@ -58,7 +58,8 @@ public class ysearch {
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
if (promoteSearchPageGreeting.length() == 0) promoteSearchPageGreeting = "P2P WEB SEARCH";
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
// get query
String querystring = (post == null) ? "" : post.get("search", "").trim();
final serverObjects prop = new serverObjects();
@ -143,7 +144,24 @@ public class ysearch {
// patch until better search profiles are available
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
if (post.get("cat", "href").equals("href")) {
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 3000)).size() > 1) try {
Thread.sleep(3000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 60000)).size() > 12) try {
Thread.sleep(10000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
Thread.sleep(30000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if ((!block) && (post.get("cat", "href").equals("href"))) {
final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
@ -166,7 +184,6 @@ public class ysearch {
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search
String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
TreeSet<String> queryHashes = indexWord.words2hashes(query[0]);
plasmaSearchQuery theQuery = new plasmaSearchQuery(
querystring,
@ -226,10 +243,10 @@ public class ysearch {
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
sb.localSearches.add(theQuery);
TreeSet<Long> handles = sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(theQuery.handle);
sb.localSearchTracker.put(client, handles);
// update the search tracker
trackerHandles.add(theQuery.handle);
sb.localSearchTracker.put(client, trackerHandles);
int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
prop.put("num-results_offset", offset);

31
htroot/yacysearch.java
Unescape View File

@ -93,11 +93,11 @@ public class yacysearch {
// get query
String querystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); // SRU compliance
final serverObjects prop = new serverObjects();
boolean rss = (post == null) ? false : post.get("rss", "false").equals("true");
if ((post == null) || (env == null) || (querystring.length() == 0) || (!searchAllowed)) {
// we create empty entries for template strings
final serverObjects prop = new serverObjects();
prop.put("searchagain", "0");
prop.put("input", input);
prop.put("display", display);
@ -174,8 +174,24 @@ public class yacysearch {
// patch until better search profiles are available
if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
serverObjects prop = new serverObjects();
if (post.get("cat", "href").equals("href")) {
// check the search tracker
TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
boolean block = false;
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 3000)).size() > 1) try {
Thread.sleep(3000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 60000)).size() > 12) try {
Thread.sleep(10000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
Thread.sleep(30000);
block = true;
} catch (InterruptedException e) { e.printStackTrace(); }
if ((!block) && (post.get("cat", "href").equals("href"))) {
final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
@ -302,12 +318,11 @@ public class yacysearch {
theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
sb.localSearches.add(theQuery);
TreeSet<Long> handles = sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet<Long>();
handles.add(theQuery.handle);
sb.localSearchTracker.put(client, handles);
prop = new serverObjects();
// update the search tracker
trackerHandles.add(theQuery.handle);
sb.localSearchTracker.put(client, trackerHandles);
int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
prop.put("num-results_offset", offset);
prop.put("num-results_itemscount", "0");

Write Preview
Loading…
Cancel
Save