From 9c989fe5f7f33a880ec3b174a0bb941ea6fa65d1 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sat, 15 Mar 2008 00:49:16 +0000 Subject: [PATCH] fixed deadlock git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4562 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/http/httpc.java | 43 +++++----- .../http/httpdByteCountInputStream.java | 1 - .../de/anomic/plasma/plasmaCrawlBalancer.java | 1 + source/de/anomic/plasma/plasmaCrawlNURL.java | 6 +- .../de/anomic/plasma/plasmaSearchEvent.java | 4 +- .../de/anomic/plasma/plasmaSwitchboard.java | 2 +- source/de/anomic/plasma/plasmaWordIndex.java | 78 ++++++------------- source/de/anomic/server/serverCore.java | 2 + source/de/anomic/yacy/yacyClient.java | 6 +- 9 files changed, 53 insertions(+), 90 deletions(-) diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index 2fa84e22d..02bda454c 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -395,12 +395,14 @@ public final class httpc { // trying to establish a connection to the address this.initTime = System.currentTimeMillis(); this.lastIO = System.currentTimeMillis(); - this.socket.setKeepAlive(false); + this.socket.setKeepAlive(true); // set socket timeout and keep alive behavior assert timeout >= 1000; this.socket.setSoTimeout(timeout); // waiting time for read this.socket.setTcpNoDelay(true); // no accumulation until buffer is full this.socket.setSoLinger(true, timeout); // wait for all data being written on close() + this.socket.setSendBufferSize(1440); // read http://www.cisco.com/warp/public/105/38.shtml + this.socket.setReceiveBufferSize(1440); // read http://www.cisco.com/warp/public/105/38.shtml // get the connection this.socket.connect(address, timeout); @@ -1411,8 +1413,9 @@ public final class httpc { * * @param procOS * @param file + * @throws IOException */ - public void writeContent(Object procOS, File file) { + public void writeContent(Object procOS, File file) throws IOException { // this writes the input stream to either another output stream or // a file or both. FileOutputStream bufferOS = null; @@ -1421,22 +1424,18 @@ public final class httpc { } catch (FileNotFoundException e) { file = null; } - try { - InputStream is = this.getContentInputStream(); - if (procOS == null) { - writeX(is, null, bufferOS); - } else if (procOS instanceof OutputStream) { - writeX(is, (OutputStream) procOS, bufferOS); - //writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, bufferOS); - } else if (procOS instanceof Writer) { - String charSet = this.responseHeader.getCharacterEncoding(); - if (charSet == null) charSet = httpHeader.DEFAULT_CHARSET; - writeX(is, charSet, (Writer) procOS, bufferOS, charSet); - } else { - throw new IllegalArgumentException("Invalid procOS object type '" + procOS.getClass().getName() + "'"); - } - } catch (IOException e) { - e.printStackTrace(); + InputStream is = this.getContentInputStream(); + if (procOS == null) { + writeX(is, null, bufferOS); + } else if (procOS instanceof OutputStream) { + writeX(is, (OutputStream) procOS, bufferOS); + //writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, bufferOS); + } else if (procOS instanceof Writer) { + String charSet = this.responseHeader.getCharacterEncoding(); + if (charSet == null) charSet = httpHeader.DEFAULT_CHARSET; + writeX(is, charSet, (Writer) procOS, bufferOS, charSet); + } else { + throw new IllegalArgumentException("Invalid procOS object type '" + procOS.getClass().getName() + "'"); } if (bufferOS != null) { @@ -1451,12 +1450,12 @@ public final class httpc { } - public void writeX(InputStream source, OutputStream procOS, OutputStream bufferOS) { + public void writeX(InputStream source, OutputStream procOS, OutputStream bufferOS) throws IOException { byte[] buffer = new byte[2048]; int l, c = 0; lastIO = System.currentTimeMillis(); - io: while (true) try { + io: while (true) { l = source.read(buffer, 0, buffer.length); if (l < 0) break; if (l == 0) try { @@ -1470,10 +1469,6 @@ public final class httpc { c += l; if (procOS != null) procOS.write(buffer, 0, l); if (bufferOS != null) bufferOS.write(buffer, 0, l); - } catch (IOException e) { - System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes. cause: " + e.getMessage()); - e.printStackTrace(); - break; } // flush the streams diff --git a/source/de/anomic/http/httpdByteCountInputStream.java b/source/de/anomic/http/httpdByteCountInputStream.java index 54339bcb7..6004f9dec 100644 --- a/source/de/anomic/http/httpdByteCountInputStream.java +++ b/source/de/anomic/http/httpdByteCountInputStream.java @@ -96,7 +96,6 @@ public class httpdByteCountInputStream extends FilterInputStream { if (readCount > 0) this.byteCount += readCount; return readCount; } catch (IOException e) { - e.printStackTrace(); throw new IOException(e.getMessage() + "; b.length = " + b.length + ", off = " + off + ", len = " + len); } } diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java index 235301f27..c04b2e7ce 100644 --- a/source/de/anomic/plasma/plasmaCrawlBalancer.java +++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java @@ -118,6 +118,7 @@ public class plasmaCrawlBalancer { public synchronized void close() { while (domainStacksNotEmpty()) flushOnceDomStacks(0, true); // flush to ram, because the ram flush is optimized + size(); try { flushAllRamStack(); } catch (IOException e) {} if (urlFileIndex != null) { urlFileIndex.close(); diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index 244a8d1f9..6152c4849 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -62,9 +62,9 @@ public class plasmaCrawlNURL { public static final int STACK_TYPE_MOVIE = 12; // put on movie stack public static final int STACK_TYPE_MUSIC = 13; // put on music stack - private static final long minimumLocalDelta = 50; // the minimum time difference between access of the same local domain - private static final long minimumGlobalDelta = 500; // the minimum time difference between access of the same global domain - private static final long maximumDomAge = 60000; // the maximum age of a domain until it is used for another crawl attempt + private static final long minimumLocalDelta = 10; // the minimum time difference between access of the same local domain + private static final long minimumGlobalDelta = 333; // the minimum time difference between access of the same global domain + private static final long maximumDomAge = 60000; // the maximum age of a domain until it is used for another crawl attempt private plasmaCrawlBalancer coreStack; // links found by crawling to depth-1 private plasmaCrawlBalancer limitStack; // links found by crawling at target depth diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index 01ecd77b6..9ce724d23 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -245,9 +245,7 @@ public final class plasmaSearchEvent { // sort the local containers and truncate it to a limited count, // so following sortings together with the global results will be fast - synchronized (rankedCache) { - rankedCache.execQuery(); - } + rankedCache.execQuery(); } } diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 5a600446c..b56afabb7 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -1719,6 +1719,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing"); // closing all still running db importer jobs this.dbImportManager.close(); + httpc.closeAllConnections(); crawlQueues.close(); wikiDB.close(); blogDB.close(); @@ -1733,7 +1734,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser parser.close(); plasmaHTCache.close(); sbQueue.close(); - httpc.closeAllConnections(); webStructure.flushCitationReference("crg"); webStructure.close(); log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager (stand by...)"); diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index bbe618e1c..07a340c6d 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -340,24 +340,18 @@ public final class plasmaWordIndex implements indexRI { // get from cache indexContainer container; - synchronized (dhtOutCache) { - container = dhtOutCache.getContainer(wordHash, urlselection); - } - synchronized (dhtInCache) { - if (container == null) { - container = dhtInCache.getContainer(wordHash, urlselection); - } else { - container.addAllUnique(dhtInCache.getContainer(wordHash, urlselection)); - } + container = dhtOutCache.getContainer(wordHash, urlselection); + if (container == null) { + container = dhtInCache.getContainer(wordHash, urlselection); + } else { + container.addAllUnique(dhtInCache.getContainer(wordHash, urlselection)); } // get from collection index - synchronized (this) { - if (container == null) { - container = collections.getContainer(wordHash, urlselection); - } else { - container.addAllUnique(collections.getContainer(wordHash, urlselection)); - } + if (container == null) { + container = collections.getContainer(wordHash, urlselection); + } else { + container.addAllUnique(collections.getContainer(wordHash, urlselection)); } if (container == null) return null; @@ -443,10 +437,8 @@ public final class plasmaWordIndex implements indexRI { public void close() { dhtInCache.close(); dhtOutCache.close(); - synchronized (this) { - collections.close(); - loadedURL.close(); - } + collections.close(); + loadedURL.close(); } public indexContainer deleteContainer(String wordHash) { @@ -455,29 +447,17 @@ public final class plasmaWordIndex implements indexRI { indexRWIRowEntry.urlEntryRow, dhtInCache.sizeContainer(wordHash) + dhtOutCache.sizeContainer(wordHash) + collections.indexSize(wordHash) ); - synchronized (dhtInCache) { - c.addAllUnique(dhtInCache.deleteContainer(wordHash)); - } - synchronized (dhtOutCache) { - c.addAllUnique(dhtOutCache.deleteContainer(wordHash)); - } - synchronized (this) { - c.addAllUnique(collections.deleteContainer(wordHash)); - } + c.addAllUnique(dhtInCache.deleteContainer(wordHash)); + c.addAllUnique(dhtOutCache.deleteContainer(wordHash)); + c.addAllUnique(collections.deleteContainer(wordHash)); return c; } public boolean removeEntry(String wordHash, String urlHash) { boolean removed = false; - synchronized (dhtInCache) { - removed = removed | (dhtInCache.removeEntry(wordHash, urlHash)); - } - synchronized (dhtOutCache) { - removed = removed | (dhtOutCache.removeEntry(wordHash, urlHash)); - } - synchronized (this) { - removed = removed | (collections.removeEntry(wordHash, urlHash)); - } + removed = removed | (dhtInCache.removeEntry(wordHash, urlHash)); + removed = removed | (dhtOutCache.removeEntry(wordHash, urlHash)); + removed = removed | (collections.removeEntry(wordHash, urlHash)); return removed; } @@ -494,29 +474,17 @@ public final class plasmaWordIndex implements indexRI { public int removeEntries(String wordHash, Set urlHashes) { int removed = 0; - synchronized (dhtInCache) { - removed += dhtInCache.removeEntries(wordHash, urlHashes); - } - synchronized (dhtOutCache) { - removed += dhtOutCache.removeEntries(wordHash, urlHashes); - } - synchronized (this) { - removed += collections.removeEntries(wordHash, urlHashes); - } + removed += dhtInCache.removeEntries(wordHash, urlHashes); + removed += dhtOutCache.removeEntries(wordHash, urlHashes); + removed += collections.removeEntries(wordHash, urlHashes); return removed; } public String removeEntriesExpl(String wordHash, Set urlHashes) { String removed = ""; - synchronized (dhtInCache) { - removed += dhtInCache.removeEntries(wordHash, urlHashes) + ", "; - } - synchronized (dhtOutCache) { - removed += dhtOutCache.removeEntries(wordHash, urlHashes) + ", "; - } - synchronized (this) { - removed += collections.removeEntries(wordHash, urlHashes); - } + removed += dhtInCache.removeEntries(wordHash, urlHashes) + ", "; + removed += dhtOutCache.removeEntries(wordHash, urlHashes) + ", "; + removed += collections.removeEntries(wordHash, urlHashes); return removed; } diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 5027f2181..a877c83c0 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -423,7 +423,9 @@ public final class serverCore extends serverAbstractThread implements serverThre controlSocket.setSoLinger(true, this.timeout); // ensure that MTU-48 is not exceeded to prevent that routers cannot handle large data packets + // read http://www.cisco.com/warp/public/105/38.shtml for explanation controlSocket.setSendBufferSize(1440); + controlSocket.setReceiveBufferSize(1440); // create session Session connection = new Session(sessionThreadGroup, controlSocket, this.timeout); diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 74d85dcf0..1c03e142c 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -99,7 +99,7 @@ public final class yacyClient { HashMap result = null; final serverObjects post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), null); - for (int retry = 0; retry < 3; retry++) try { + for (int retry = 0; retry < 4; retry++) try { // generate request post.put("count", "20"); post.put("seed", yacyCore.seedDB.mySeed().genSeedStr(post.get("key", ""))); @@ -119,10 +119,10 @@ public final class yacyClient { break; } catch (Exception e) { if (Thread.currentThread().isInterrupted()) { - yacyCore.log.logFine("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' interrupted."); + yacyCore.log.logWarning("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' interrupted."); return -1; } else { - yacyCore.log.logFine("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' exception: " + e.getMessage() + "; retry = " + retry); // here VERY OFTEN a 'Connection reset' appears. What is the cause? + yacyCore.log.logWarning("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' exception: " + e.getMessage() + "; retry = " + retry); // here VERY OFTEN a 'Connection reset' appears. What is the cause? // try again (go into loop) } result = null;