From d2ac0aa682f9522a79829d794be81ea6a48a875b Mon Sep 17 00:00:00 2001 From: orbiter Date: Sat, 2 May 2009 01:40:03 +0000 Subject: [PATCH] - fixed possible bugs in Stack (may affect Crawler reset) and RandomAccess handling - increased default memory size to 180MB - fixed possible bug in http client reset (there was a deadlock) - bug in BOBHeap marked, but not solved, cause is still unknown. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5912 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- defaults/yacy.init | 4 +- htroot/Threaddump_p.java | 2 + source/de/anomic/crawler/IndexingStack.java | 37 ++++++++++++++----- source/de/anomic/http/httpClient.java | 7 +++- source/de/anomic/kelondro/blob/BLOBHeap.java | 2 +- .../kelondro/io/CachedRandomAccess.java | 6 +-- .../de/anomic/kelondro/text/IODispatcher.java | 8 ++++ startYACY.bat | 2 +- startYACY.sh | 2 +- 9 files changed, 51 insertions(+), 19 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index fde9dd867..40713818c 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -619,8 +619,8 @@ cleanup.deletionPublishedNews = true # -Xmx and -Xms maximum/init Java heap size # both values should be equal, # othervise the YaCy-internal memory supervision does not work -javastart_Xmx=Xmx120m -javastart_Xms=Xms120m +javastart_Xmx=Xmx180m +javastart_Xms=Xms180m # priority of the yacy-process # is valid in unix/shell and windows environments but diff --git a/htroot/Threaddump_p.java b/htroot/Threaddump_p.java index 553fc509e..eb7fcb067 100644 --- a/htroot/Threaddump_p.java +++ b/htroot/Threaddump_p.java @@ -40,6 +40,7 @@ import java.util.HashMap; import de.anomic.htmlFilter.htmlFilterCharacterCoding; import de.anomic.http.httpRequestHeader; import de.anomic.kelondro.util.FileUtils; +import de.anomic.kelondro.util.MemoryControl; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -78,6 +79,7 @@ public class Threaddump_p { final ArrayList> traces = new ArrayList>(); for (int i = 0; i < multipleCount; i++) { traces.add(Thread.getAllStackTraces()); + if (MemoryControl.available() < 20 * 1024 * 1024) break; } appendStackTraceStats(sb.getRootPath(), buffer, traces, plain, Thread.State.BLOCKED); appendStackTraceStats(sb.getRootPath(), buffer, traces, plain, Thread.State.RUNNABLE); diff --git a/source/de/anomic/crawler/IndexingStack.java b/source/de/anomic/crawler/IndexingStack.java index aaaa12726..0b4411d6f 100644 --- a/source/de/anomic/crawler/IndexingStack.java +++ b/source/de/anomic/crawler/IndexingStack.java @@ -95,16 +95,33 @@ public class IndexingStack { } public synchronized QueueEntry pop() throws IOException { - if (sbQueueStack.size() == 0) { - Log.logInfo("IndexingStack", "sbQueueStack.size() == 0"); - return null; - } - final Row.Entry b = sbQueueStack.pot(); - if (b == null) { - Log.logInfo("IndexingStack", "sbQueueStack.pot() == null"); - return null; - } - return new QueueEntry(b); + synchronized (sbQueueStack) { + int sizeBefore; + while ((sizeBefore = sbQueueStack.size()) > 0) { + Row.Entry b = sbQueueStack.pot(); + if (b == null) { + Log.logInfo("IndexingStack", "sbQueueStack.pot() == null"); + if (sbQueueStack.size() < sizeBefore) continue; + Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pot() == null; trying pop()"); + } + if (sbQueueStack.size() < sizeBefore) { + return new QueueEntry(b); + } else { + Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pot() != null; trying pop()"); + } + sizeBefore = sbQueueStack.size(); + b = sbQueueStack.pop(); + if (b == null) { + Log.logInfo("IndexingStack", "sbQueueStack.pop() == null"); + if (sbQueueStack.size() < sizeBefore) continue; + Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pop() == null; failed"); + return null; + } + return new QueueEntry(b); + } + Log.logInfo("IndexingStack", "sbQueueStack.size() == 0"); + return null; + } } public synchronized QueueEntry remove(final String urlHash) { diff --git a/source/de/anomic/http/httpClient.java b/source/de/anomic/http/httpClient.java index a869c393f..f787fe660 100644 --- a/source/de/anomic/http/httpClient.java +++ b/source/de/anomic/http/httpClient.java @@ -90,6 +90,7 @@ public class httpClient { /** * set options for client */ + MultiThreadedHttpConnectionManager.shutdownAll(); initConnectionManager(); // accept self-signed or untrusted certificates @@ -112,7 +113,11 @@ public class httpClient { } public static void initConnectionManager() { - MultiThreadedHttpConnectionManager.shutdownAll(); + if (conManager != null) { + conManager.closeIdleConnections(0); + conManager.deleteClosedConnections(); + conManager.shutdown(); + } conManager = new MultiThreadedHttpConnectionManager(); apacheHttpClient = new HttpClient(conManager); diff --git a/source/de/anomic/kelondro/blob/BLOBHeap.java b/source/de/anomic/kelondro/blob/BLOBHeap.java index 8faa25a92..e896c2d76 100755 --- a/source/de/anomic/kelondro/blob/BLOBHeap.java +++ b/source/de/anomic/kelondro/blob/BLOBHeap.java @@ -175,7 +175,7 @@ public final class BLOBHeap extends BLOBHeapModifier implements BLOB { assert posBuffer + 4 + key.length <= ba.length : "posBuffer = " + posBuffer + ", key.length = " + key.length + ", ba.length = " + ba.length; System.arraycopy(key, 0, ba, posBuffer + 4, key.length); assert posBuffer + 4 + key.length + blob.length <= ba.length : "posBuffer = " + posBuffer + ", key.length = " + key.length + ", blob.length = " + blob.length + ", ba.length = " + ba.length; - System.arraycopy(blob, 0, ba, posBuffer + 4 + key.length, blob.length); + System.arraycopy(blob, 0, ba, posBuffer + 4 + key.length, blob.length); //java.lang.ArrayIndexOutOfBoundsException here posFile += 4 + key.length + blob.length; posBuffer += 4 + key.length + blob.length; } diff --git a/source/de/anomic/kelondro/io/CachedRandomAccess.java b/source/de/anomic/kelondro/io/CachedRandomAccess.java index 16f641f63..d287d665f 100644 --- a/source/de/anomic/kelondro/io/CachedRandomAccess.java +++ b/source/de/anomic/kelondro/io/CachedRandomAccess.java @@ -72,8 +72,8 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra return; } // we fill the cache here - int available = (int) (this.RAFile.length() - seek); - if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len); + long available = this.RAFile.length() - seek; + if (available < (long) len) throw new IOException("EOF, available = " + available + ", requested = " + len); if (cachestart + cachelen == seek && cache.length - cachelen >= len) { RAFile.readFully(cache, cachelen, len); //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes"); @@ -81,7 +81,7 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra cachelen += len; } else { // fill the cache as much as possible - int m = Math.min(available, cache.length); + int m = (int) Math.min(available, (long) cache.length); RAFile.readFully(cache, 0, m); cachestart = seek; cachelen = m; diff --git a/source/de/anomic/kelondro/text/IODispatcher.java b/source/de/anomic/kelondro/text/IODispatcher.java index 9b8bc8c8f..a4ba373f1 100644 --- a/source/de/anomic/kelondro/text/IODispatcher.java +++ b/source/de/anomic/kelondro/text/IODispatcher.java @@ -209,6 +209,14 @@ public class IODispatcher extends Thread { } public File merge() { + if (!f1.exists()) { + Log.logWarning("IODispatcher", "merge of file (1) " + f1.getName() + " failed: file does not exists"); + return null; + } + if (!f2.exists()) { + Log.logWarning("IODispatcher", "merge of file (2) " + f2.getName() + " failed: file does not exists"); + return null; + } try { return array.mergeMount(f1, f2, factory, payloadrow, newFile); } catch (IOException e) { diff --git a/startYACY.bat b/startYACY.bat index c4eb3f32f..d20f90a31 100644 --- a/startYACY.bat +++ b/startYACY.bat @@ -15,7 +15,7 @@ For %%X in (libx/*.jar) Do Call %0 CPGEN libx\%%X REM Please change the "javastart" settings in the web-interface "Basic Configuration" -> "Advanced" set jmx= set jms= -set javacmd=-Xmx120m -Xms120m +set javacmd=-Xmx180m -Xms180m set priolvl=10 set priority=/BELOWNORMAL if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX diff --git a/startYACY.sh b/startYACY.sh index 87ff4e2ef..2f0415559 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -155,7 +155,7 @@ then # JAVA_ARGS="-$i $JAVA_ARGS"; # done else - JAVA_ARGS="-Xmx120m -Xms120m $JAVA_ARGS"; + JAVA_ARGS="-Xmx180m -Xms180m $JAVA_ARGS"; PORT="8080" fi