|
|
|
@ -451,16 +451,16 @@ public class Balancer {
|
|
|
|
|
// this is only to protection against the worst case, where the crawler could
|
|
|
|
|
// behave in a DoS-manner
|
|
|
|
|
Log.logInfo("BALANCER", "forcing crawl-delay of " + sleeptime + " milliseconds for " + crawlEntry.url().getHost() + ": " + Latency.waitingRemainingExplain(crawlEntry.url(), minimumLocalDelta, minimumGlobalDelta) + ", top.size() = " + top.size() + ", delayed.size() = " + delayed.size() + ", domainStacks.size() = " + domainStacks.size() + ", domainStacksInitSize = " + this.domStackInitSize);
|
|
|
|
|
long loops = sleeptime / 3000;
|
|
|
|
|
long rest = sleeptime % 3000;
|
|
|
|
|
long loops = sleeptime / 1000;
|
|
|
|
|
long rest = sleeptime % 1000;
|
|
|
|
|
if (loops < 2) {
|
|
|
|
|
rest = rest + 3000 * loops;
|
|
|
|
|
rest = rest + 1000 * loops;
|
|
|
|
|
loops = 0;
|
|
|
|
|
}
|
|
|
|
|
if (rest > 0) {try {synchronized(this) { this.wait(rest); }} catch (final InterruptedException e) {}}
|
|
|
|
|
if (rest > 0) {try {this.wait(rest); } catch (final InterruptedException e) {}}
|
|
|
|
|
for (int i = 0; i < loops; i++) {
|
|
|
|
|
Log.logInfo("BALANCER", "waiting for " + crawlEntry.url().getHost() + ": " + ((loops - i) * 3) + " seconds remaining...");
|
|
|
|
|
try {synchronized(this) { this.wait(3000); }} catch (final InterruptedException e) {}
|
|
|
|
|
try {this.wait(1000); } catch (final InterruptedException e) {}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
this.ddc.remove(crawlEntry.url().hash());
|
|
|
|
|