added and fixed time-out behaviour during search

see also: http://www.yacy-forum.de/viewtopic.php?p=19823#19823

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1986 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/plasma/plasmaSearchEvent.java

@@ -218,13 +218,15 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
         // attention: if minEntries is too high, this method will not terminate within the maxTime
 
         plasmaWordIndexEntryContainer searchResult = new plasmaWordIndexEntryContainer(null);
-        searchResult.add(rcLocal);
-        searchResult.add(rcGlobal);
-        
         long preorderTime = profileLocal.getTargetTime(plasmaSearchTimingProfile.PROCESS_PRESORT);
         long postorderTime = profileLocal.getTargetTime(plasmaSearchTimingProfile.PROCESS_POSTSORT);
         
         profileLocal.startTimer();
+        long pst = System.currentTimeMillis();
+        searchResult.add(rcLocal, preorderTime / 3);
+        searchResult.add(rcGlobal, preorderTime / 3);
+        preorderTime = preorderTime - (System.currentTimeMillis() - pst);
+        if (preorderTime < 0) preorderTime = 100;
         plasmaSearchPreOrder preorder = new plasmaSearchPreOrder(query, ranking);
         preorder.addContainer(searchResult, preorderTime);
         profileLocal.setYieldTime(plasmaSearchTimingProfile.PROCESS_PRESORT);
@@ -237,12 +239,12 @@ public final class plasmaSearchEvent extends Thread implements Runnable {
         
         // start url-fetch
         plasmaWordIndexEntry entry;
-        long postorderLimitTime = (postorderTime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + postorderTime;
+        long postorderLimitTime = (postorderTime < 0) ? Long.MAX_VALUE : (System.currentTimeMillis() + postorderTime);
         plasmaCrawlLURL.Entry page;
         int minEntries = profileLocal.getTargetCount(plasmaSearchTimingProfile.PROCESS_POSTSORT);
         try {
             while (preorder.hasNext()) {
-                if ((acc.sizeFetched() >= minEntries) && (System.currentTimeMillis() >= postorderLimitTime)) break;
+                if ((acc.sizeFetched() >= minEntries) || (System.currentTimeMillis() >= postorderLimitTime)) break;
                 entry = preorder.next();
                 // find the url entry
                 try {

source/de/anomic/plasma/plasmaWordIndex.java

@@ -279,18 +279,17 @@ public final class plasmaWordIndex {
         // e.g. indexTransfer might keep this container for minutes while
         // several new pages could be added to the index, possibly with the same words that have
         // been selected for transfer
-        container.add(ramCache.getContainer(wordHash, true));
+        container.add(ramCache.getContainer(wordHash, true), maxTime / 2);
 
         // get from assortments
-        container.add(assortmentCluster.getFromAll(wordHash, (maxTime < 0) ? -1 : maxTime / 2));
+        container.add(assortmentCluster.getFromAll(wordHash, (maxTime < 0) ? -1 : maxTime / 2), maxTime / 2);
 
         // get from backend
         if (maxTime > 0) {
             maxTime = maxTime - (System.currentTimeMillis() - start);
-            if (maxTime < 0)
+            if (maxTime < 0) maxTime = 100;
-                maxTime = 100;
         }
-        container.add(backend.getContainer(wordHash, deleteIfEmpty, (maxTime < 0) ? -1 : maxTime));
+        container.add(backend.getContainer(wordHash, deleteIfEmpty, (maxTime < 0) ? -1 : maxTime / 2), maxTime / 2);
         return container;
     }
 

source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java

@@ -102,7 +102,7 @@ public final class plasmaWordIndexAssortmentCluster {
         if (newContainer.size() > clusterCount) return newContainer; // it will not fit
         plasmaWordIndexEntryContainer buffer;
         while ((buffer = assortments[newContainer.size() - 1].remove(wordHash)) != null) {
-            if (newContainer.add(buffer) == 0) return newContainer; // security check; othervise this loop does not terminate
+            if (newContainer.add(buffer, -1) == 0) return newContainer; // security check; othervise this loop does not terminate
             if (newContainer.size() > clusterCount) return newContainer; // it will not fit
         }
         // the assortment (newContainer.size() - 1) should now be empty. put it in there
@@ -200,7 +200,7 @@ public final class plasmaWordIndexAssortmentCluster {
         if (newContainer == null) return null;
         
         // clean up the whole thing and try to insert the container then
-        newContainer.add(removeFromAll(wordHash, -1));
+        newContainer.add(removeFromAll(wordHash, -1), -1);
         if (newContainer.size() > clusterCapacity) return newContainer;
         storeStretched(wordHash, newContainer);
         return null;
@@ -210,10 +210,12 @@ public final class plasmaWordIndexAssortmentCluster {
         // removes all records from all the assortments and return them
         plasmaWordIndexEntryContainer buffer, record = new plasmaWordIndexEntryContainer(wordHash);
         long limitTime = (maxTime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxTime;
+        long remainingTime;
         for (int i = 0; i < clusterCount; i++) {
             buffer = assortments[i].remove(wordHash);
-            if (buffer != null) record.add(buffer);
+            remainingTime = limitTime - System.currentTimeMillis();
-            if (System.currentTimeMillis() > limitTime) break;
+            if (0 > remainingTime) break;
+            if (buffer != null) record.add(buffer, remainingTime);
         }
         return record;
     }
@@ -222,10 +224,13 @@ public final class plasmaWordIndexAssortmentCluster {
         // collect all records from all the assortments and return them
         plasmaWordIndexEntryContainer buffer, record = new plasmaWordIndexEntryContainer(wordHash);
         long limitTime = (maxTime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxTime;
+        long remainingTime;
         for (int i = 0; i < clusterCount; i++) {
             buffer = assortments[i].get(wordHash);
-            if (buffer != null) record.add(buffer);
+            remainingTime = limitTime - System.currentTimeMillis();
-            if (System.currentTimeMillis() > limitTime) break;
+            if (0 > remainingTime) break;
+            if (buffer != null) record.add(buffer, remainingTime);
+            
         }
         return record;
     }

source/de/anomic/plasma/plasmaWordIndexCache.java

@@ -439,7 +439,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
             String wordHash = container.wordHash();
             plasmaWordIndexEntryContainer entries = (plasmaWordIndexEntryContainer) wCache.get(wordHash); // null pointer exception? wordhash != null! must be cache==null
             if (entries == null) entries = new plasmaWordIndexEntryContainer(wordHash);
-            added = entries.add(container);
+            added = entries.add(container, -1);
             if (added > 0) {
                 wCache.put(wordHash, entries);
                 hashScore.addScore(wordHash, added);

source/de/anomic/plasma/plasmaWordIndexClassicDB.java

@@ -181,14 +181,15 @@ public class plasmaWordIndexClassicDB {
         }
     }
 
-    public plasmaWordIndexEntryContainer getContainer(String wordHash, boolean deleteIfEmpty, long maxTime) {
+    public synchronized plasmaWordIndexEntryContainer getContainer(String wordHash, boolean deleteIfEmpty, long maxTime) {
         long start = System.currentTimeMillis();
+        if ((maxTime < 0) || (maxTime > 60000)) maxTime=60000; // maximum is one minute
         if (plasmaWordIndexEntity.wordHash2path(databaseRoot, wordHash).exists()) {
             plasmaWordIndexEntity entity = this.getEntity(wordHash, deleteIfEmpty, (maxTime < 0) ? -1 : maxTime * 9 / 10);
             plasmaWordIndexEntryContainer container = new plasmaWordIndexEntryContainer(wordHash);
             plasmaWordIndexEntry entry;
             Iterator i = entity.elements(true);
-            while ((i.hasNext()) && ((maxTime < 0) || (System.currentTimeMillis() < start + maxTime))) {
+            while ((i.hasNext()) && (System.currentTimeMillis() < (start + maxTime))) {
                 entry = (plasmaWordIndexEntry) i.next();
                 container.add(entry);
             }

source/de/anomic/plasma/plasmaWordIndexEntryContainer.java

@@ -116,12 +116,13 @@ public final class plasmaWordIndexEntryContainer {
         return c;
     }
     
-    public int add(plasmaWordIndexEntryContainer c) {
+    public int add(plasmaWordIndexEntryContainer c, long maxTime) {
         // returns the number of new elements
+        long startTime = System.currentTimeMillis();
         if (c == null) return 0;
         Iterator i = c.entries();
         int x = 0;
-        while (i.hasNext()) {
+        while ((i.hasNext()) && ((maxTime < 0) || ((startTime + maxTime) > System.currentTimeMillis()))) {
             try {
                 if (addi((plasmaWordIndexEntry) i.next())) x++;
             } catch (ConcurrentModificationException e) {}
@@ -131,12 +132,13 @@ public final class plasmaWordIndexEntryContainer {
     }
 
     private boolean addi(plasmaWordIndexEntry entry) {
-        // returns true if the new entry was added, false if it already existet
+        // returns true if the new entry was added, false if it already existed
-        plasmaWordIndexEntry oldEntry = (plasmaWordIndexEntry) container.get(entry.getUrlHash());
+        plasmaWordIndexEntry oldEntry = (plasmaWordIndexEntry) container.put(entry.getUrlHash(), entry);
         if ((oldEntry != null) && (entry.isOlder(oldEntry))) { // A more recent Entry is already in this container
+            container.put(entry.getUrlHash(), oldEntry); // put it back
             return false;
         }
-        return (container.put(entry.getUrlHash(), entry) == null);
+        return (oldEntry == null);
     }
 
     public boolean contains(String urlHash) {

source/de/anomic/yacy/yacyClient.java

@@ -507,7 +507,7 @@ public final class yacyClient {
             }
 
             // finally insert the containers to the index
-            for (int m = 0; m < words; m++) { containerCache.add(container[m]); }
+            for (int m = 0; m < words; m++) { containerCache.add(container[m], -1); }
 
             // generate statistics
             long searchtime;