- disabled domZones (buggy, must think about better solution)

- increased time-out for dns resolver and isLocal property

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7233 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent ec38eca278
commit fcd40cd30f

@ -212,12 +212,8 @@ public class Crawler_p {
if (crawlingStartURL!= null &&(crawlingStartURL.isFile() || crawlingStartURL.isSMB())) storeHTCache = false; if (crawlingStartURL!= null &&(crawlingStartURL.isFile() || crawlingStartURL.isSMB())) storeHTCache = false;
env.setConfig("storeHTCache", (storeHTCache) ? "true" : "false"); env.setConfig("storeHTCache", (storeHTCache) ? "true" : "false");
final String cachePolicyString = post.get("cachePolicy", "iffresh"); CrawlProfile.CacheStrategy cachePolicy = CrawlProfile.CacheStrategy.parse(post.get("cachePolicy", "iffresh"));
CrawlProfile.CacheStrategy cachePolicy = CrawlProfile.CacheStrategy.IFFRESH; if (cachePolicy == null) cachePolicy = CrawlProfile.CacheStrategy.IFFRESH;
if (cachePolicyString.equals("nocache")) cachePolicy = CrawlProfile.CacheStrategy.NOCACHE;
if (cachePolicyString.equals("iffresh")) cachePolicy = CrawlProfile.CacheStrategy.IFFRESH;
if (cachePolicyString.equals("ifexist")) cachePolicy = CrawlProfile.CacheStrategy.IFEXIST;
if (cachePolicyString.equals("cacheonly")) cachePolicy = CrawlProfile.CacheStrategy.CACHEONLY;
final boolean xsstopw = post.get("xsstopw", "off").equals("on"); final boolean xsstopw = post.get("xsstopw", "off").equals("on");
env.setConfig("xsstopw", (xsstopw) ? "true" : "false"); env.setConfig("xsstopw", (xsstopw) ? "true" : "false");

@ -192,7 +192,9 @@ public final class RankingProcess extends Thread {
} }
// check constraints // check constraints
if (!testFlags(iEntry)) continue; if (!testFlags(iEntry)) {
continue;
}
// check document domain // check document domain
if (query.contentdom != ContentDomain.TEXT) { if (query.contentdom != ContentDomain.TEXT) {
@ -203,10 +205,12 @@ public final class RankingProcess extends Thread {
} }
// check tld domain // check tld domain
/*
if ((DigestURI.domDomain(iEntry.metadataHash()) & this.query.zonecode) == 0) { if ((DigestURI.domDomain(iEntry.metadataHash()) & this.query.zonecode) == 0) {
// filter out all tld that do not match with wanted tld domain // filter out all tld that do not match with wanted tld domain
continue; continue;
} }
*/
// count domZones // count domZones
//this.domZones[DigestURI.domDomain(iEntry.metadataHash())]++; //this.domZones[DigestURI.domDomain(iEntry.metadataHash())]++;

@ -462,7 +462,7 @@ public class Domains {
// call i.getHostName() using concurrency to interrupt execution in case of a time-out // call i.getHostName() using concurrency to interrupt execution in case of a time-out
try { try {
return TimeoutRequest.getHostName(i, 500); return TimeoutRequest.getHostName(i, 1000);
} catch (ExecutionException e) { } catch (ExecutionException e) {
return i.getHostAddress(); return i.getHostAddress();
} }
@ -484,7 +484,7 @@ public class Domains {
// call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out // call dnsResolveNetBased(host) using concurrency to interrupt execution in case of a time-out
try { try {
boolean doCaching = true; boolean doCaching = true;
ip = TimeoutRequest.getByName(host, 500); // this makes the DNS request to backbone ip = TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone
if ((ip == null) || if ((ip == null) ||
(ip.isLoopbackAddress()) || (ip.isLoopbackAddress()) ||
(nameCacheNoCachingList.containsKey(host)) (nameCacheNoCachingList.containsKey(host))

Loading…
Cancel
Save