boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for deadlocks in crawler

Browse Source
pull/1/head
Michael Peter Christen 11 years ago
parent 7a2f3e2353
commit 9c6228d948
4 changed files with 96 additions and 71 deletions
Show Stats Download Patch File Download Diff File

153
source/net/yacy/crawler/HostBalancer.java
Unescape View File

@ -201,17 +201,19 @@ public class HostBalancer implements Balancer {
* @throws SpaceExceededException
*/
@Override
public synchronized String push(final Request entry, CrawlProfile profile, final RobotsTxt robots) throws IOException, SpaceExceededException {
public String push(final Request entry, CrawlProfile profile, final RobotsTxt robots) throws IOException, SpaceExceededException {
if (this.has(entry.url().hash())) return "double occurrence";
depthCache.put(entry.url().hash(), entry.depth());
String hosthash = ASCII.String(entry.url().hash(), 6, 6);
HostQueue queue = this.queues.get(hosthash);
if (queue == null) {
queue = new HostQueue(this.hostsPath, entry.url().getHost(), entry.url().getPort(), this.queues.size() > 100, this.exceed134217727);
this.queues.put(hosthash, queue);
robots.ensureExist(entry.url(), profile.getAgent(), true); // concurrently load all robots.txt
synchronized (this) {
HostQueue queue = this.queues.get(hosthash);
if (queue == null) {
queue = new HostQueue(this.hostsPath, entry.url().getHost(), entry.url().getPort(), this.queues.size() > 100, this.exceed134217727);
this.queues.put(hosthash, queue);
robots.ensureExist(entry.url(), profile.getAgent(), true); // concurrently load all robots.txt
}
return queue.push(entry, profile, robots);
}
return queue.push(entry, profile, robots);
}
/**
@ -227,80 +229,93 @@ public class HostBalancer implements Balancer {
* @throws SpaceExceededException
*/
@Override
public synchronized Request pop(boolean delay, CrawlSwitchboard cs, RobotsTxt robots) throws IOException {
public Request pop(boolean delay, CrawlSwitchboard cs, RobotsTxt robots) throws IOException {
tryagain: while (true) try {
if (this.roundRobinHostHashes.size() == 0) {
// refresh the round-robin cache
this.roundRobinHostHashes.addAll(this.queues.keySet());
// quickly get rid of small stacks to reduce number of files:
if (this.roundRobinHostHashes.size() > 100) {
// if there are stacks with less than 10 entries, remove all stacks with more than 10 entries
// this shall kick out small stacks to prevent that too many files are opened for very wide crawls
boolean smallStacksExist = false;
boolean singletonStacksExist = false;
smallsearch: for (String s: this.roundRobinHostHashes) {
HostQueue hq = this.queues.get(s);
if (hq != null) {
int size = hq.size();
if (size == 1) {singletonStacksExist = true; break smallsearch;}
if (size <= 10) {smallStacksExist = true; break smallsearch;}
}
}
if (singletonStacksExist) {
Iterator<String> i = this.roundRobinHostHashes.iterator();
while (i.hasNext()) {
String s = i.next();
HostQueue rhq = null;
String rhh = null;
synchronized (this) {
if (this.roundRobinHostHashes.size() == 0) {
// refresh the round-robin cache
this.roundRobinHostHashes.addAll(this.queues.keySet());
// quickly get rid of small stacks to reduce number of files:
if (this.roundRobinHostHashes.size() > 100) {
// if there are stacks with less than 10 entries, remove all stacks with more than 10 entries
// this shall kick out small stacks to prevent that too many files are opened for very wide crawls
boolean smallStacksExist = false;
boolean singletonStacksExist = false;
smallsearch: for (String s: this.roundRobinHostHashes) {
HostQueue hq = this.queues.get(s);
if (hq == null || hq.size() != 1) {i.remove();}
if (hq != null) {
int size = hq.size();
if (size == 1) {singletonStacksExist = true; break smallsearch;}
if (size <= 10) {smallStacksExist = true; break smallsearch;}
}
}
} else if (smallStacksExist) {
Iterator<String> i = this.roundRobinHostHashes.iterator();
while (i.hasNext()) {
String s = i.next();
HostQueue hq = this.queues.get(s);
if (hq == null || hq.size() > 10) {i.remove();}
if (singletonStacksExist) {
Iterator<String> i = this.roundRobinHostHashes.iterator();
while (i.hasNext()) {
String s = i.next();
HostQueue hq = this.queues.get(s);
if (hq == null || hq.size() != 1) {i.remove();}
}
} else if (smallStacksExist) {
Iterator<String> i = this.roundRobinHostHashes.iterator();
while (i.hasNext()) {
String s = i.next();
HostQueue hq = this.queues.get(s);
if (hq == null || hq.size() > 10) {i.remove();}
}
}
}
}
}
if (this.roundRobinHostHashes.size() == 0) return null;
// first strategy: get one entry which does not need sleep time
for (String nextHH: this.roundRobinHostHashes) {
HostQueue hq = this.queues.get(nextHH);
int delta = Latency.waitingRemainingGuessed(hq.getHost(), DigestURL.hosthash(hq.getHost(), hq.getPort()), robots, ClientIdentification.yacyInternetCrawlerAgent);
if (delta <= 10 || this.roundRobinHostHashes.size() == 1) {
this.roundRobinHostHashes.remove(nextHH);
Request request = hq == null ? null : hq.pop(delay, cs, robots);
int size = hq == null ? 0 : hq.size();
if (size == 0) {
hq.close();
this.queues.remove(nextHH);
if (this.roundRobinHostHashes.size() == 0) return null;
// first strategy: get one entry which does not need sleep time
Iterator<String> nhhi = this.roundRobinHostHashes.iterator();
nosleep: while (nhhi.hasNext()) {
rhh = nhhi.next();
rhq = this.queues.get(rhh);
if (rhq == null) {
nhhi.remove();
continue nosleep;
}
int delta = Latency.waitingRemainingGuessed(rhq.getHost(), DigestURL.hosthash(rhq.getHost(), rhq.getPort()), robots, ClientIdentification.yacyInternetCrawlerAgent);
if (delta <= 10 || this.roundRobinHostHashes.size() == 1) {
nhhi.remove();
break nosleep;
}
if (request != null) return request;
}
}
// second strategy: take from the largest stack and clean round robin cache
int largest = Integer.MIN_VALUE;
String nextHH = null;
for (String h: this.roundRobinHostHashes) {
HostQueue hq = this.queues.get(h);
if (hq != null) {
int s = hq.size();
if (s > largest) {
largest = s;
nextHH = h;
if (rhq == null) {
// second strategy: take from the largest stack and clean round robin cache
int largest = Integer.MIN_VALUE;
for (String h: this.roundRobinHostHashes) {
HostQueue hq = this.queues.get(h);
if (hq != null) {
int s = hq.size();
if (s > largest) {
largest = s;
rhh = h;
}
}
}
this.roundRobinHostHashes.clear(); // start from the beginning next time
rhq = this.queues.get(rhh);
}
}
this.roundRobinHostHashes.clear(); // start from the beginning next time
HostQueue hq = this.queues.get(nextHH);
Request request = hq == null ? null : hq.pop(delay, cs, robots);
if (hq != null && hq.size() == 0) {
hq.close();
this.queues.remove(nextHH);
if (rhq == null) continue tryagain;
Request request = rhq.pop(delay, cs, robots); // this pop is outside of synchronization to prevent blocking of pushes
int size = rhq.size();
if (size == 0) {
synchronized (this) {
this.queues.remove(rhh);
}
rhq.close();
}
if (request == null) continue tryagain;
return request;
} catch (ConcurrentModificationException e) {
continue tryagain;

2
source/net/yacy/crawler/HostQueue.java
Unescape View File

@ -253,7 +253,7 @@ public class HostQueue implements Balancer {
}
this.depthStacks.clear();
String[] l = this.hostPath.list();
for (String s: l) {
if (l != null) for (String s: l) {
new File(this.hostPath, s).delete();
}
this.hostPath.delete();

2
source/net/yacy/crawler/data/CrawlQueues.java
Unescape View File

@ -393,7 +393,7 @@ public class CrawlQueues {
*/
private String loadIsPossible(final StackType stackType) {
//System.out.println("stacksize = " + noticeURL.stackSize(stackType));
if (this.noticeURL.stackSize(stackType) == 0) {
if (this.noticeURL.isEmpty(stackType)) {
//log.logDebug("GlobalCrawl: queue is empty");
return "stack is empty";
}

10
source/net/yacy/crawler/data/NoticedURL.java
Unescape View File

@ -126,6 +126,16 @@ public class NoticedURL {
return true;
}
public boolean isEmpty(final StackType stackType) {
switch (stackType) {
case NOLOAD: return (this.noloadStack == null) ? true : this.noloadStack.isEmpty();
case LOCAL: return (this.coreStack == null) ? true : this.coreStack.isEmpty();
case GLOBAL: return (this.limitStack == null) ? true : this.limitStack.isEmpty();
case REMOTE: return (this.remoteStack == null) ? true : this.remoteStack.isEmpty();
default: return true;
}
}
public int stackSize(final StackType stackType) {
switch (stackType) {
case NOLOAD: return (this.noloadStack == null) ? 0 : this.noloadStack.size();

Write Preview
Loading…
Cancel
Save