- disabled domZones (buggy, must think about better solution)

- increased time-out for dns resolver and isLocal property

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7233 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent ec38eca278
commit fcd40cd30f

@ -212,12 +212,8 @@ public class Crawler_p {
if (crawlingStartURL!= null &&(crawlingStartURL.isFile() || crawlingStartURL.isSMB())) storeHTCache = false;
env.setConfig("storeHTCache", (storeHTCache) ? "true" : "false");
final String cachePolicyString = post.get("cachePolicy", "iffresh");
CrawlProfile.CacheStrategy cachePolicy = CrawlProfile.CacheStrategy.IFFRESH;
if (cachePolicyString.equals("nocache")) cachePolicy = CrawlProfile.CacheStrategy.NOCACHE;
if (cachePolicyString.equals("iffresh")) cachePolicy = CrawlProfile.CacheStrategy.IFFRESH;
if (cachePolicyString.equals("ifexist")) cachePolicy = CrawlProfile.CacheStrategy.IFEXIST;
if (cachePolicyString.equals("cacheonly")) cachePolicy = CrawlProfile.CacheStrategy.CACHEONLY;
CrawlProfile.CacheStrategy cachePolicy = CrawlProfile.CacheStrategy.parse(post.get("cachePolicy", "iffresh"));
if (cachePolicy == null) cachePolicy = CrawlProfile.CacheStrategy.IFFRESH;
final boolean xsstopw = post.get("xsstopw", "off").equals("on");
env.setConfig("xsstopw", (xsstopw) ? "true" : "false");

@ -192,7 +192,9 @@ public final class RankingProcess extends Thread {
}
// check constraints
if (!testFlags(iEntry)) continue;
if (!testFlags(iEntry)) {
continue;
}
// check document domain
if (query.contentdom != ContentDomain.TEXT) {
@ -203,10 +205,12 @@ public final class RankingProcess extends Thread {
}
// check tld domain
/*
if ((DigestURI.domDomain(iEntry.metadataHash()) & this.query.zonecode) == 0) {
// filter out all tld that do not match with wanted tld domain
continue;
}
*/
// count domZones
//this.domZones[DigestURI.domDomain(iEntry.metadataHash())]++;

@ -462,7 +462,7 @@ public class Domains {
// call i.getHostName() using concurrency to interrupt execution in case of a time-out
try {
return TimeoutRequest.getHostName(i, 500);
return TimeoutRequest.getHostName(i, 1000);
} catch (ExecutionException e) {
return i.getHostAddress();
}
@ -484,7 +484,7 @@ public class Domains {
// call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out
try {
boolean doCaching = true;
ip = TimeoutRequest.getByName(host, 500); // this makes the DNS request to backbone
ip = TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone
if ((ip == null) ||
(ip.isLoopbackAddress()) ||
(nameCacheNoCachingList.containsKey(host))

Loading…
Cancel
Save