From 33fb2f756d31ec30f25a7d188d9768bcf8ebc9c3 Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Thu, 4 Oct 2007 10:40:30 +0000
Subject: [PATCH] added emergency fail case in remote crawls in extreme
 situations this will cause that no remote crawls are send out any more this
 is bad, but it protects the case where failing remote crawls fill up the
 local queue too much, which is even worse

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4141 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 .../de/anomic/plasma/plasmaSwitchboard.java   | 35 ++++++++++---------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java
index 4c0f6cd3b..5c14cdd4c 100644
--- a/source/de/anomic/plasma/plasmaSwitchboard.java
+++ b/source/de/anomic/plasma/plasmaSwitchboard.java
@@ -2237,10 +2237,24 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
             if (robinsonPrivateCase) return false;
         }
         
-        // if the server is busy, we do crawling more slowly
-        //if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
+        // check local indexing queues
+        // in case the placing of remote crawl fails, there must be space in the local queue to work off the remote crawl
+        if (sbQueue.size() >= indexingSlots * 2) {
+            log.logFine("LimitCrawl: too many processes in indexing queue, dismissed (" +
+            "sbQueueSize=" + sbQueue.size() + ")");
+            return false;
+        }
+        if (cacheLoader.size() >= crawlSlots) {
+            log.logFine("LimitCrawl: too many processes in loader queue, dismissed (" +
+            "cacheLoader=" + cacheLoader.size() + ")");
+            return false;
+        }
+        if (onlineCaution()) {
+            log.logFine("LimitCrawl: online caution, omitting processing");
+            return false;
+        }
         
-        // if crawling was paused we have to wait until we wer notified to continue
+        // if crawling was paused we have to wait until we were notified to continue
         Object[] status = (Object[])this.crawlJobsStatus.get(CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
         synchronized(status[CRAWLJOB_SYNC]) {
             if (((Boolean)status[CRAWLJOB_STATUS]).booleanValue()) {
@@ -2277,20 +2291,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
                 if (success) return true;
             }
 
-            processLocalCrawling(urlEntry, profile, stats); // emergency case
-            
-            if (sbQueue.size() >= indexingSlots) {
-                log.logFine("LimitCrawl: too many processes in indexing queue, delayed to protect emergency case (" +
-                "sbQueueSize=" + sbQueue.size() + ")");
-                return false;
-            }
-            
-            if (cacheLoader.size() >= crawlSlots) {
-                log.logFine("LimitCrawl: too many processes in loader queue, delayed to protect emergency case (" +
-                "cacheLoader=" + cacheLoader.size() + ")");
-                return false;
-            }
-            
+            processLocalCrawling(urlEntry, profile, stats); // emergency case, work off the crawl locally            
             return true;
         } catch (IOException e) {
             log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);