- fixed possible bugs in Stack (may affect Crawler reset) and RandomAccess handling

- increased default memory size to 180MB
- fixed possible bug in http client reset (there was a deadlock)
- bug in BOBHeap marked, but not solved, cause is still unknown.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5912 6c8d7289-2bf4-0310-a012-ef5d649a1542

defaults/yacy.init

@@ -619,8 +619,8 @@ cleanup.deletionPublishedNews = true
 # -Xmx<size> and -Xms<size> maximum/init Java heap size
 # both values should be equal,
 # othervise the YaCy-internal memory supervision does not work
-javastart_Xmx=Xmx120m
-javastart_Xms=Xms120m
+javastart_Xmx=Xmx180m
+javastart_Xms=Xms180m
 
 # priority of the yacy-process
 # is valid in unix/shell and windows environments but

htroot/Threaddump_p.java

@@ -40,6 +40,7 @@ import java.util.HashMap;
 import de.anomic.htmlFilter.htmlFilterCharacterCoding;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.kelondro.util.FileUtils;
+import de.anomic.kelondro.util.MemoryControl;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
@@ -78,6 +79,7 @@ public class Threaddump_p {
             final ArrayList<Map<Thread,StackTraceElement[]>> traces = new ArrayList<Map<Thread,StackTraceElement[]>>();
             for (int i = 0; i < multipleCount; i++) {
                 traces.add(Thread.getAllStackTraces());
+                if (MemoryControl.available() < 20 * 1024 * 1024) break;
             }
             appendStackTraceStats(sb.getRootPath(), buffer, traces, plain, Thread.State.BLOCKED);
             appendStackTraceStats(sb.getRootPath(), buffer, traces, plain, Thread.State.RUNNABLE);

source/de/anomic/crawler/IndexingStack.java

@@ -95,17 +95,34 @@ public class IndexingStack {
     }
 
     public synchronized QueueEntry pop() throws IOException {
-        if (sbQueueStack.size() == 0) {
-        	Log.logInfo("IndexingStack", "sbQueueStack.size() == 0");
-        	return null;
-        }
-        final Row.Entry b = sbQueueStack.pot();
+    	synchronized (sbQueueStack) {
+    		int sizeBefore;
+	    	while ((sizeBefore = sbQueueStack.size()) > 0) {
+		        Row.Entry b = sbQueueStack.pot();
 		        if (b == null) {
 		        	Log.logInfo("IndexingStack", "sbQueueStack.pot() == null");
+		        	if (sbQueueStack.size() < sizeBefore) continue;
+		        	Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pot() == null; trying pop()");
+		        }
+		        if (sbQueueStack.size() < sizeBefore) {
+		        	return new QueueEntry(b);
+		        } else {
+		        	Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pot() != null; trying pop()");
+		        }
+		        sizeBefore = sbQueueStack.size();
+		        b = sbQueueStack.pop();
+		        if (b == null) {
+		        	Log.logInfo("IndexingStack", "sbQueueStack.pop() == null");
+		        	if (sbQueueStack.size() < sizeBefore) continue;
+		        	Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pop() == null; failed");
 		        	return null;
 		        }
 		        return new QueueEntry(b);
 	    	}
+	    	Log.logInfo("IndexingStack", "sbQueueStack.size() == 0");
+	        return null;
+    	}
+    }
 
     public synchronized QueueEntry remove(final String urlHash) {
         final Iterator<Row.Entry> i = sbQueueStack.stackIterator(true);

source/de/anomic/http/httpClient.java

@@ -90,6 +90,7 @@ public class httpClient {
         /**
          * set options for client
          */
+        MultiThreadedHttpConnectionManager.shutdownAll();
         initConnectionManager();
         
         // accept self-signed or untrusted certificates
@@ -112,7 +113,11 @@ public class httpClient {
     }
 
     public static void initConnectionManager() {
-        MultiThreadedHttpConnectionManager.shutdownAll();
+        if (conManager != null) {
+        	conManager.closeIdleConnections(0);
+        	conManager.deleteClosedConnections();
+        	conManager.shutdown();
+        }
         conManager = new MultiThreadedHttpConnectionManager();
         apacheHttpClient = new HttpClient(conManager);
         

source/de/anomic/kelondro/blob/BLOBHeap.java

@@ -175,7 +175,7 @@ public final class BLOBHeap extends BLOBHeapModifier implements BLOB {
             assert posBuffer + 4 + key.length <= ba.length : "posBuffer = " + posBuffer + ", key.length = " + key.length + ", ba.length = " + ba.length;
             System.arraycopy(key, 0, ba, posBuffer + 4, key.length);
             assert posBuffer + 4 + key.length + blob.length <= ba.length : "posBuffer = " + posBuffer + ", key.length = " + key.length + ", blob.length = " + blob.length + ", ba.length = " + ba.length;
-            System.arraycopy(blob, 0, ba, posBuffer + 4 + key.length, blob.length);
+            System.arraycopy(blob, 0, ba, posBuffer + 4 + key.length, blob.length); //java.lang.ArrayIndexOutOfBoundsException here
             posFile += 4 + key.length + blob.length;
             posBuffer += 4 + key.length + blob.length;
         }

source/de/anomic/kelondro/io/CachedRandomAccess.java

@@ -72,8 +72,8 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
             return;
         }
         // we fill the cache here
-        int available = (int) (this.RAFile.length() - seek);
-        if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len);
+        long available = this.RAFile.length() - seek;
+        if (available < (long) len) throw new IOException("EOF, available = " + available + ", requested = " + len);
         if (cachestart + cachelen == seek && cache.length - cachelen >= len) {
             RAFile.readFully(cache, cachelen, len);
             //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes");
@@ -81,7 +81,7 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
             cachelen += len;
         } else {
             // fill the cache as much as possible
-            int m = Math.min(available, cache.length);
+            int m = (int) Math.min(available, (long) cache.length);
             RAFile.readFully(cache, 0, m);
             cachestart = seek;
             cachelen = m;

source/de/anomic/kelondro/text/IODispatcher.java

@@ -209,6 +209,14 @@ public class IODispatcher <ReferenceType extends Reference> extends Thread {
         }
 
         public File merge() {
+        	if (!f1.exists()) {
+        		Log.logWarning("IODispatcher", "merge of file (1) " + f1.getName() + " failed: file does not exists");
+        		return null;
+        	}
+        	if (!f2.exists()) {
+        		Log.logWarning("IODispatcher", "merge of file (2) " + f2.getName() + " failed: file does not exists");
+        		return null;
+        	}
             try {
                 return array.mergeMount(f1, f2, factory, payloadrow, newFile);
             } catch (IOException e) {

startYACY.bat

@@ -15,7 +15,7 @@ For %%X in (libx/*.jar) Do Call %0 CPGEN libx\%%X
 REM Please change the "javastart" settings in the web-interface "Basic Configuration" -> "Advanced" 
 set jmx=
 set jms=
-set javacmd=-Xmx120m -Xms120m
+set javacmd=-Xmx180m -Xms180m
 set priolvl=10
 set priority=/BELOWNORMAL
 if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX

startYACY.sh

@@ -155,7 +155,7 @@ then
 #		JAVA_ARGS="-$i $JAVA_ARGS";
 #	done
 else
-    JAVA_ARGS="-Xmx120m -Xms120m $JAVA_ARGS";
+    JAVA_ARGS="-Xmx180m -Xms180m $JAVA_ARGS";
     PORT="8080"
 fi