boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed deadlock

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4562 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent c565906050
commit 9c989fe5f7
9 changed files with 53 additions and 90 deletions
Show Stats Download Patch File Download Diff File

19
source/de/anomic/http/httpc.java
Unescape View File

@ -395,12 +395,14 @@ public final class httpc {
// trying to establish a connection to the address
this.initTime = System.currentTimeMillis();
this.lastIO = System.currentTimeMillis();
this.socket.setKeepAlive(false);
this.socket.setKeepAlive(true);
// set socket timeout and keep alive behavior
assert timeout >= 1000;
this.socket.setSoTimeout(timeout); // waiting time for read
this.socket.setTcpNoDelay(true); // no accumulation until buffer is full
this.socket.setSoLinger(true, timeout); // wait for all data being written on close()
this.socket.setSendBufferSize(1440); // read http://www.cisco.com/warp/public/105/38.shtml
this.socket.setReceiveBufferSize(1440); // read http://www.cisco.com/warp/public/105/38.shtml
// get the connection
this.socket.connect(address, timeout);
@ -1411,8 +1413,9 @@ public final class httpc {
*
* @param procOS
* @param file
* @throws IOException
*/
public void writeContent(Object procOS, File file) {
public void writeContent(Object procOS, File file) throws IOException {
// this writes the input stream to either another output stream or
// a file or both.
FileOutputStream bufferOS = null;
@ -1421,7 +1424,6 @@ public final class httpc {
} catch (FileNotFoundException e) {
file = null;
}
try {
InputStream is = this.getContentInputStream();
if (procOS == null) {
writeX(is, null, bufferOS);
@ -1435,9 +1437,6 @@ public final class httpc {
} else {
throw new IllegalArgumentException("Invalid procOS object type '" + procOS.getClass().getName() + "'");
}
} catch (IOException e) {
e.printStackTrace();
}
if (bufferOS != null) {
try {
@ -1451,12 +1450,12 @@ public final class httpc {
}
public void writeX(InputStream source, OutputStream procOS, OutputStream bufferOS) {
public void writeX(InputStream source, OutputStream procOS, OutputStream bufferOS) throws IOException {
byte[] buffer = new byte[2048];
int l, c = 0;
lastIO = System.currentTimeMillis();
io: while (true) try {
io: while (true) {
l = source.read(buffer, 0, buffer.length);
if (l < 0) break;
if (l == 0) try {
@ -1470,10 +1469,6 @@ public final class httpc {
c += l;
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
} catch (IOException e) {
System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes. cause: " + e.getMessage());
e.printStackTrace();
break;
}
// flush the streams

1
source/de/anomic/http/httpdByteCountInputStream.java
Unescape View File

@ -96,7 +96,6 @@ public class httpdByteCountInputStream extends FilterInputStream {
if (readCount > 0) this.byteCount += readCount;
return readCount;
} catch (IOException e) {
e.printStackTrace();
throw new IOException(e.getMessage() + "; b.length = " + b.length + ", off = " + off + ", len = " + len);
}
}

1
source/de/anomic/plasma/plasmaCrawlBalancer.java
Unescape View File

@ -118,6 +118,7 @@ public class plasmaCrawlBalancer {
public synchronized void close() {
while (domainStacksNotEmpty()) flushOnceDomStacks(0, true); // flush to ram, because the ram flush is optimized
size();
try { flushAllRamStack(); } catch (IOException e) {}
if (urlFileIndex != null) {
urlFileIndex.close();

4
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -62,8 +62,8 @@ public class plasmaCrawlNURL {
public static final int STACK_TYPE_MOVIE = 12; // put on movie stack
public static final int STACK_TYPE_MUSIC = 13; // put on music stack
private static final long minimumLocalDelta = 50; // the minimum time difference between access of the same local domain
private static final long minimumGlobalDelta = 500; // the minimum time difference between access of the same global domain
private static final long minimumLocalDelta = 10; // the minimum time difference between access of the same local domain
private static final long minimumGlobalDelta = 333; // the minimum time difference between access of the same global domain
private static final long maximumDomAge = 60000; // the maximum age of a domain until it is used for another crawl attempt
private plasmaCrawlBalancer coreStack; // links found by crawling to depth-1

2
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -245,11 +245,9 @@ public final class plasmaSearchEvent {
// sort the local containers and truncate it to a limited count,
// so following sortings together with the global results will be fast
synchronized (rankedCache) {
rankedCache.execQuery();
}
}
}
public static void cleanupEvents(boolean all) {
// remove old events in the event cache

2
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -1719,6 +1719,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing");
// closing all still running db importer jobs
this.dbImportManager.close();
httpc.closeAllConnections();
crawlQueues.close();
wikiDB.close();
blogDB.close();
@ -1733,7 +1734,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
parser.close();
plasmaHTCache.close();
sbQueue.close();
httpc.closeAllConnections();
webStructure.flushCitationReference("crg");
webStructure.close();
log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager (stand by...)");

32
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -340,25 +340,19 @@ public final class plasmaWordIndex implements indexRI {
// get from cache
indexContainer container;
synchronized (dhtOutCache) {
container = dhtOutCache.getContainer(wordHash, urlselection);
}
synchronized (dhtInCache) {
if (container == null) {
container = dhtInCache.getContainer(wordHash, urlselection);
} else {
container.addAllUnique(dhtInCache.getContainer(wordHash, urlselection));
}
}
// get from collection index
synchronized (this) {
if (container == null) {
container = collections.getContainer(wordHash, urlselection);
} else {
container.addAllUnique(collections.getContainer(wordHash, urlselection));
}
}
if (container == null) return null;
@ -443,11 +437,9 @@ public final class plasmaWordIndex implements indexRI {
public void close() {
dhtInCache.close();
dhtOutCache.close();
synchronized (this) {
collections.close();
loadedURL.close();
}
}
public indexContainer deleteContainer(String wordHash) {
indexContainer c = new indexContainer(
@ -455,29 +447,17 @@ public final class plasmaWordIndex implements indexRI {
indexRWIRowEntry.urlEntryRow,
dhtInCache.sizeContainer(wordHash) + dhtOutCache.sizeContainer(wordHash) + collections.indexSize(wordHash)
);
synchronized (dhtInCache) {
c.addAllUnique(dhtInCache.deleteContainer(wordHash));
}
synchronized (dhtOutCache) {
c.addAllUnique(dhtOutCache.deleteContainer(wordHash));
}
synchronized (this) {
c.addAllUnique(collections.deleteContainer(wordHash));
}
return c;
}
public boolean removeEntry(String wordHash, String urlHash) {
boolean removed = false;
synchronized (dhtInCache) {
removed = removed | (dhtInCache.removeEntry(wordHash, urlHash));
}
synchronized (dhtOutCache) {
removed = removed | (dhtOutCache.removeEntry(wordHash, urlHash));
}
synchronized (this) {
removed = removed | (collections.removeEntry(wordHash, urlHash));
}
return removed;
}
@ -494,29 +474,17 @@ public final class plasmaWordIndex implements indexRI {
public int removeEntries(String wordHash, Set<String> urlHashes) {
int removed = 0;
synchronized (dhtInCache) {
removed += dhtInCache.removeEntries(wordHash, urlHashes);
}
synchronized (dhtOutCache) {
removed += dhtOutCache.removeEntries(wordHash, urlHashes);
}
synchronized (this) {
removed += collections.removeEntries(wordHash, urlHashes);
}
return removed;
}
public String removeEntriesExpl(String wordHash, Set<String> urlHashes) {
String removed = "";
synchronized (dhtInCache) {
removed += dhtInCache.removeEntries(wordHash, urlHashes) + ", ";
}
synchronized (dhtOutCache) {
removed += dhtOutCache.removeEntries(wordHash, urlHashes) + ", ";
}
synchronized (this) {
removed += collections.removeEntries(wordHash, urlHashes);
}
return removed;
}

2
source/de/anomic/server/serverCore.java
Unescape View File

@ -423,7 +423,9 @@ public final class serverCore extends serverAbstractThread implements serverThre
controlSocket.setSoLinger(true, this.timeout);
// ensure that MTU-48 is not exceeded to prevent that routers cannot handle large data packets
// read http://www.cisco.com/warp/public/105/38.shtml for explanation
controlSocket.setSendBufferSize(1440);
controlSocket.setReceiveBufferSize(1440);
// create session
Session connection = new Session(sessionThreadGroup, controlSocket, this.timeout);

6
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -99,7 +99,7 @@ public final class yacyClient {
HashMap<String, String> result = null;
final serverObjects post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), null);
for (int retry = 0; retry < 3; retry++) try {
for (int retry = 0; retry < 4; retry++) try {
// generate request
post.put("count", "20");
post.put("seed", yacyCore.seedDB.mySeed().genSeedStr(post.get("key", "")));
@ -119,10 +119,10 @@ public final class yacyClient {
break;
} catch (Exception e) {
if (Thread.currentThread().isInterrupted()) {
yacyCore.log.logFine("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' interrupted.");
yacyCore.log.logWarning("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' interrupted.");
return -1;
} else {
yacyCore.log.logFine("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' exception: " + e.getMessage() + "; retry = " + retry); // here VERY OFTEN a 'Connection reset' appears. What is the cause?
yacyCore.log.logWarning("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' exception: " + e.getMessage() + "; retry = " + retry); // here VERY OFTEN a 'Connection reset' appears. What is the cause?
// try again (go into loop)
}
result = null;

Write Preview
Loading…
Cancel
Save