prevent too high search request frequency submitted from the same peer

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4813 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/yacy/search.java

@@ -121,6 +121,29 @@ public final class search {
         	return prop;
         }
         
+        // check the search tracker
+        TreeSet<Long> trackerHandles = sb.remoteSearchTracker.get(client);
+        if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
+        boolean block = false;
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() -   6000)).size() >  1) try {
+            Thread.sleep(3000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() -  60000)).size() > 12) try {
+            Thread.sleep(10000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
+            Thread.sleep(30000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (block) {
+            prop.put("links", "");
+            prop.put("linkcount", "0");
+            prop.put("references", "");
+            return prop;
+        }
+        
         // tell all threads to do nothing for a specific time
         sb.intermissionAllThreads(3000);
 
@@ -291,10 +314,11 @@ public final class search {
         theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
         theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime();
         sb.remoteSearches.add(theQuery);
-        TreeSet<Long> handles = sb.remoteSearchTracker.get(client);
+        
-        if (handles == null) handles = new TreeSet<Long>();
+        // update the search tracker
-        handles.add(theQuery.handle);
+        trackerHandles.add(theQuery.handle);
-        sb.remoteSearchTracker.put(client, handles);
+        sb.remoteSearchTracker.put(client, trackerHandles);
+        
         
         // log
         yacyCore.log.logInfo("EXIT HASH SEARCH: " +

htroot/yacy/ui/result.java

@@ -58,6 +58,7 @@ public class result {
         String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
         if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
         if (promoteSearchPageGreeting.length() == 0) promoteSearchPageGreeting = "P2P WEB SEARCH";
+        String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
         
         // get query
         String querystring = (post == null) ? "" : post.get("search", "").trim();
@@ -143,7 +144,24 @@ public class result {
         // patch until better search profiles are available
         if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
         
-        if (post.get("cat", "href").equals("href")) {
+        // check the search tracker
+        TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
+        if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
+        boolean block = false;
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() -   3000)).size() >  1) try {
+            Thread.sleep(3000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() -  60000)).size() > 12) try {
+            Thread.sleep(10000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
+            Thread.sleep(30000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        
+        if ((!block) && (post.get("cat", "href").equals("href"))) {
 
             final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
             boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
@@ -166,7 +184,6 @@ public class result {
             final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
         
             // do the search
-            String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
             TreeSet<String> queryHashes = indexWord.words2hashes(query[0]);
             plasmaSearchQuery theQuery = new plasmaSearchQuery(
         			querystring,
@@ -226,10 +243,10 @@ public class result {
             theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
             theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
             sb.localSearches.add(theQuery);
-            TreeSet<Long> handles = sb.localSearchTracker.get(client);
+
-            if (handles == null) handles = new TreeSet<Long>();
+            // update the search tracker
-            handles.add(theQuery.handle);
+            trackerHandles.add(theQuery.handle);
-            sb.localSearchTracker.put(client, handles);
+            sb.localSearchTracker.put(client, trackerHandles);
             
             int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
             prop.put("num-results_offset", offset);

htroot/yacy/user/ysearch.java

@@ -58,6 +58,7 @@ public class ysearch {
         String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
         if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", "");
         if (promoteSearchPageGreeting.length() == 0) promoteSearchPageGreeting = "P2P WEB SEARCH";
+        String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
         
         // get query
         String querystring = (post == null) ? "" : post.get("search", "").trim();
@@ -143,7 +144,24 @@ public class ysearch {
         // patch until better search profiles are available
         if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
         
-        if (post.get("cat", "href").equals("href")) {
+        // check the search tracker
+        TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
+        if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
+        boolean block = false;
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() -   3000)).size() >  1) try {
+            Thread.sleep(3000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() -  60000)).size() > 12) try {
+            Thread.sleep(10000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
+            Thread.sleep(30000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        
+        if ((!block) && (post.get("cat", "href").equals("href"))) {
 
             final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
             boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
@@ -166,7 +184,6 @@ public class ysearch {
             final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
         
             // do the search
-            String client = (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP); // the search client who initiated the search
             TreeSet<String> queryHashes = indexWord.words2hashes(query[0]);
             plasmaSearchQuery theQuery = new plasmaSearchQuery(
         			querystring,
@@ -226,10 +243,10 @@ public class ysearch {
             theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
             theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
             sb.localSearches.add(theQuery);
-            TreeSet<Long> handles = sb.localSearchTracker.get(client);
+            
-            if (handles == null) handles = new TreeSet<Long>();
+            // update the search tracker
-            handles.add(theQuery.handle);
+            trackerHandles.add(theQuery.handle);
-            sb.localSearchTracker.put(client, handles);
+            sb.localSearchTracker.put(client, trackerHandles);
             
             int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
             prop.put("num-results_offset", offset);

htroot/yacysearch.java

@@ -93,11 +93,11 @@ public class yacysearch {
         
         // get query
         String querystring = (post == null) ? "" : post.get("query", post.get("search", "")).trim(); // SRU compliance
+        final serverObjects prop = new serverObjects();
         
         boolean rss = (post == null) ? false : post.get("rss", "false").equals("true");
         if ((post == null) || (env == null) || (querystring.length() == 0) || (!searchAllowed)) {
             // we create empty entries for template strings
-            final serverObjects prop = new serverObjects();
             prop.put("searchagain", "0");
             prop.put("input", input);
             prop.put("display", display);
@@ -174,8 +174,24 @@ public class yacysearch {
         // patch until better search profiles are available
         if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
         
-        serverObjects prop = new serverObjects();
+        // check the search tracker
-        if (post.get("cat", "href").equals("href")) {
+        TreeSet<Long> trackerHandles = sb.localSearchTracker.get(client);
+        if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
+        boolean block = false;
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() -   3000)).size() >  1) try {
+            Thread.sleep(3000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() -  60000)).size() > 12) try {
+            Thread.sleep(10000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        if (trackerHandles.tailSet(new Long(System.currentTimeMillis() - 600000)).size() > 36) try {
+            Thread.sleep(30000);
+            block = true;
+        } catch (InterruptedException e) { e.printStackTrace(); }
+        
+        if ((!block) && (post.get("cat", "href").equals("href"))) {
 
             final TreeSet<String>[] query = plasmaSearchQuery.cleanQuery(querystring); // converts also umlaute
             boolean near = (query[0].contains("near")) && (querystring.indexOf("NEAR") >= 0);
@@ -302,12 +318,11 @@ public class yacysearch {
             theQuery.urlretrievaltime = theSearch.getURLRetrievalTime();
             theQuery.snippetcomputationtime = theSearch.getSnippetComputationTime();
             sb.localSearches.add(theQuery);
-            TreeSet<Long> handles = sb.localSearchTracker.get(client);
-            if (handles == null) handles = new TreeSet<Long>();
-            handles.add(theQuery.handle);
-            sb.localSearchTracker.put(client, handles);
             
-            prop = new serverObjects();
+            // update the search tracker
+            trackerHandles.add(theQuery.handle);
+            sb.localSearchTracker.put(client, trackerHandles);
+            
             int totalcount = theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
             prop.put("num-results_offset", offset);
             prop.put("num-results_itemscount", "0");