switched two balancer cases

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5177 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 1bbf362cef
commit 21dbb39afa

@ -446,6 +446,43 @@ public class Balancer {
// so just fill them up with some stuff // so just fill them up with some stuff
if (result == null) shiftFileToDomStacks(1000); if (result == null) shiftFileToDomStacks(1000);
// 2nd-b: check domainStacks for best match between stack size and retrieval time
String maxhash = null;
if ((result == null) && (domainStacks.size() > 0)) synchronized (domainStacks) {
// we order all domains by the number of entries per domain
// then we iterate through these domains in descending entry order
// and take that one, that has a zero waiting time
final Iterator<Map.Entry<String, LinkedList<String>>> i = domainStacks.entrySet().iterator();
Map.Entry<String, LinkedList<String>> entry;
String domhash;
LinkedList<String> domlist;
final TreeMap<Integer, String> hitlist = new TreeMap<Integer, String>();
int count = 0;
// first collect information about sizes of the domain lists
while (i.hasNext()) {
entry = i.next();
domhash = entry.getKey();
domlist = entry.getValue();
hitlist.put(Integer.valueOf(domlist.size() * 100 + count++), domhash);
}
// now iterate in descending order and fetch that one,
// that is acceptable by the minimumDelta constraint
long waitingtime;
while (hitlist.size() > 0) {
domhash = hitlist.remove(hitlist.lastKey());
if (maxhash == null) maxhash = domhash; // remember first entry
waitingtime = CrawlEntry.waitingRemainingGuessed(domhash, minimumLocalDelta, minimumGlobalDelta);
if (waitingtime < 100) {
domlist = domainStacks.get(domhash);
result = domlist.removeFirst();
if (domlist.size() == 0) domainStacks.remove(domhash);
break;
}
}
}
// 2nd-a: check domainStacks for latest arrivals // 2nd-a: check domainStacks for latest arrivals
if ((result == null) && (domainStacks.size() > 0)) synchronized (domainStacks) { if ((result == null) && (domainStacks.size() > 0)) synchronized (domainStacks) {
// we select specific domains that have not been used for a long time // we select specific domains that have not been used for a long time
@ -481,44 +518,10 @@ public class Balancer {
} }
} }
// 2nd-b: check domainStacks for best match between stack size and retrieval time // 2nd-c: if we did yet not choose any entry, we simply take that one with the most entries
if ((result == null) && (domainStacks.size() > 0)) synchronized (domainStacks) { if ((result == null) && (maxhash != null)) {
// we order all domains by the number of entries per domain LinkedList<String> domlist = domainStacks.get(maxhash);
// then we iterate through these domains in descending entry order if (domlist != null) {
// and take that one, that has a zero waiting time
final Iterator<Map.Entry<String, LinkedList<String>>> i = domainStacks.entrySet().iterator();
Map.Entry<String, LinkedList<String>> entry;
String domhash;
LinkedList<String> domlist;
final TreeMap<Integer, String> hitlist = new TreeMap<Integer, String>();
int count = 0;
// first collect information about sizes of the domain lists
while (i.hasNext()) {
entry = i.next();
domhash = entry.getKey();
domlist = entry.getValue();
hitlist.put(Integer.valueOf(domlist.size() * 100 + count++), domhash);
}
// now iterate in descending order and fetch that one,
// that is acceptable by the minimumDelta constraint
long waitingtime;
String maxhash = null;
while (hitlist.size() > 0) {
domhash = hitlist.remove(hitlist.lastKey());
if (maxhash == null) maxhash = domhash; // remember first entry
waitingtime = CrawlEntry.waitingRemainingGuessed(domhash, minimumLocalDelta, minimumGlobalDelta);
if (waitingtime == 0) {
domlist = domainStacks.get(domhash);
result = domlist.removeFirst();
if (domlist.size() == 0) domainStacks.remove(domhash);
break;
}
}
// if we did yet not choose any entry, we simply take that one with the most entries
if ((result == null) && (maxhash != null)) {
domlist = domainStacks.get(maxhash);
result = domlist.removeFirst(); result = domlist.removeFirst();
if (domlist.size() == 0) domainStacks.remove(maxhash); if (domlist.size() == 0) domainStacks.remove(maxhash);
} }

Loading…
Cancel
Save