diff --git a/source/net/yacy/cora/document/MultiProtocolURI.java b/source/net/yacy/cora/document/MultiProtocolURI.java index 3f422f927..5d9775664 100644 --- a/source/net/yacy/cora/document/MultiProtocolURI.java +++ b/source/net/yacy/cora/document/MultiProtocolURI.java @@ -2042,9 +2042,7 @@ public class MultiProtocolURI implements Serializable, Comparable 0) {try {this.wait(rest); } catch (final InterruptedException e) {}} + if (rest > 0) {try {Thread.sleep(rest);} catch (final InterruptedException e) {}} for (int i = 0; i < loops; i++) { Log.logInfo("BALANCER", "waiting for " + crawlEntry.url().getHost() + ": " + (loops - i) + " seconds remaining..."); - try {this.wait(1000); } catch (final InterruptedException e) {} + try {Thread.sleep(1000); } catch (final InterruptedException e) {} } Latency.updateAfterSelection(crawlEntry.url(), robotsTime); } @@ -488,6 +497,7 @@ public class Balancer { byte[] besturlhash = null; String besthost = null; OrderedScoreMap> nextZeroCandidates = new OrderedScoreMap>(null); + int newCandidatesForward = 10; while (i.hasNext() && nextZeroCandidates.size() < 1000) { entry = i.next(); @@ -516,7 +526,13 @@ public class Balancer { } if (w <= 0) { - nextZeroCandidates.set(new AbstractMap.SimpleEntry(entry.getKey(), urlhash), w == Integer.MIN_VALUE ? 1000 /* get new domains a chance */ : entry.getValue().size()); + if (w == Integer.MIN_VALUE && newCandidatesForward > 0) { + // give new domains a chance, but not too much; otherwise a massive downloading of robots.txt from too much domains (dns lock!) will more likely block crawling + newCandidatesForward--; + nextZeroCandidates.set(new AbstractMap.SimpleEntry(entry.getKey(), urlhash), 1000); + } else { + nextZeroCandidates.set(new AbstractMap.SimpleEntry(entry.getKey(), urlhash), entry.getValue().size()); + } } if (w < smallestWaiting || (w == smallestWaiting && this.random.nextBoolean())) { smallestWaiting = w; diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index b5f8062cc..1ffae79a9 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -374,7 +374,7 @@ public final class CrawlStacker { final DigestURI referrerURL = (entry.referrerhash() == null || entry.referrerhash().length == 0) ? null : this.nextQueue.getURL(entry.referrerhash()); // add domain to profile domain list - if (profile.domMaxPages() != Integer.MAX_VALUE) { + if (profile.domMaxPages() != Integer.MAX_VALUE && profile.domMaxPages() > 0) { profile.domInc(entry.url().getHost(), (referrerURL == null) ? null : referrerURL.getHost().toLowerCase(), entry.depth()); } @@ -478,7 +478,7 @@ public final class CrawlStacker { // deny urls that exceed allowed number of occurrences final int maxAllowedPagesPerDomain = profile.domMaxPages(); - if (maxAllowedPagesPerDomain < Integer.MAX_VALUE) { + if (maxAllowedPagesPerDomain < Integer.MAX_VALUE && maxAllowedPagesPerDomain > 0) { final DomProfile dp = profile.getDom(url.getHost()); if (dp != null && dp.count >= maxAllowedPagesPerDomain) { if (this.log.isFine()) this.log.logFine("URL '" + urlstring + "' appeared too often in crawl stack, a maximum of " + profile.domMaxPages() + " is allowed."); diff --git a/source/net/yacy/crawler/data/NoticedURL.java b/source/net/yacy/crawler/data/NoticedURL.java index 2c76050b9..6052a4be0 100644 --- a/source/net/yacy/crawler/data/NoticedURL.java +++ b/source/net/yacy/crawler/data/NoticedURL.java @@ -95,7 +95,7 @@ public class NoticedURL { this.noloadStack.clear(); } - protected synchronized void close() { + protected void close() { Log.logInfo("NoticedURL", "CLOSING ALL STACKS"); if (this.coreStack != null) { this.coreStack.close(); @@ -303,19 +303,17 @@ public class NoticedURL { int s; Request entry; int errors = 0; - synchronized (balancer) { - while ((s = balancer.size()) > 0) { - entry = balancer.pop(delay, cs, robots); - if (entry == null) { - if (s > balancer.size()) continue; - errors++; - if (errors < 100) continue; - final int aftersize = balancer.size(); - balancer.clear(); // the balancer is broken and cannot shrink - Log.logWarning("BALANCER", "entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer"); - } - return entry; + while ((s = balancer.size()) > 0) { + entry = balancer.pop(delay, cs, robots); + if (entry == null) { + if (s > balancer.size()) continue; + errors++; + if (errors < 100) continue; + final int aftersize = balancer.size(); + balancer.clear(); // the balancer is broken and cannot shrink + Log.logWarning("BALANCER", "entry is null, balancer cannot shrink (bevore pop = " + s + ", after pop = " + aftersize + "); reset of balancer"); } + return entry; } return null; } diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.java index 7846e1533..dcf6ed3aa 100644 --- a/source/net/yacy/crawler/retrieval/HTTPLoader.java +++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java @@ -126,7 +126,7 @@ public final class HTTPLoader { requestHeader.put(HeaderFramework.ACCEPT_ENCODING, this.sb.getConfig("crawler.http.acceptEncoding", DEFAULT_ENCODING)); // HTTP-Client - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); client.setRedirecting(false); // we want to handle redirection ourselves, so we don't index pages twice client.setTimout(this.socketTimeout); client.setHeader(requestHeader.entrySet()); @@ -252,7 +252,7 @@ public final class HTTPLoader { requestHeader.put(HeaderFramework.ACCEPT_CHARSET, DEFAULT_CHARSET); requestHeader.put(HeaderFramework.ACCEPT_ENCODING, DEFAULT_ENCODING); - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); client.setTimout(20000); client.setHeader(requestHeader.entrySet()); final byte[] responseBody = client.GETbytes(request.url()); diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.java index c2c8d0904..c30db0801 100644 --- a/source/net/yacy/crawler/robots/RobotsTxt.java +++ b/source/net/yacy/crawler/robots/RobotsTxt.java @@ -287,7 +287,7 @@ public class RobotsTxt { return port; } - static Object[] downloadRobotsTxt(final MultiProtocolURI robotsURL, int redirectionCount, final RobotsTxtEntry entry) throws Exception { + protected static Object[] downloadRobotsTxt(final MultiProtocolURI robotsURL, int redirectionCount, final RobotsTxtEntry entry) throws Exception { if (robotsURL == null || !robotsURL.getProtocol().startsWith("http")) return null; if (redirectionCount < 0) return new Object[]{Boolean.FALSE,null,null}; @@ -319,7 +319,7 @@ public class RobotsTxt { // setup http-client //TODO: adding Traffic statistic for robots download? - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); client.setHeader(reqHeaders.entrySet()); try { // check for interruption diff --git a/source/net/yacy/data/WorkTables.java b/source/net/yacy/data/WorkTables.java index 91ccde690..38b4edd62 100644 --- a/source/net/yacy/data/WorkTables.java +++ b/source/net/yacy/data/WorkTables.java @@ -40,6 +40,7 @@ import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.ASCII; import net.yacy.cora.document.UTF8; import net.yacy.cora.order.Base64Order; +import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.SpaceExceededException; @@ -214,7 +215,7 @@ public class WorkTables extends Tables { */ public Map execAPICalls(String host, int port, String realm, Collection pks) { // now call the api URLs and store the result status - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); client.setRealm(realm); client.setTimout(120000); Tables.Row row; @@ -246,7 +247,7 @@ public class WorkTables extends Tables { public static int execAPICall(String host, int port, String realm, String path, byte[] pk) { // now call the api URLs and store the result status - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); client.setRealm(realm); client.setTimout(120000); String url = "http://" + host + ":" + port + path; diff --git a/source/net/yacy/document/parser/sitemapParser.java b/source/net/yacy/document/parser/sitemapParser.java index 16f0a8d30..359766e0d 100644 --- a/source/net/yacy/document/parser/sitemapParser.java +++ b/source/net/yacy/document/parser/sitemapParser.java @@ -42,7 +42,6 @@ import javax.xml.parsers.DocumentBuilderFactory; import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.protocol.http.HTTPClient; @@ -113,9 +112,7 @@ public class sitemapParser extends AbstractParser implements Parser { // download document Log.logInfo("SitemapReader", "loading sitemap from " + sitemapURL.toNormalform(true)); final RequestHeader requestHeader = new RequestHeader(); - requestHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent()); - final HTTPClient client = new HTTPClient(); - client.setTimout(5000); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), 5000); client.setHeader(requestHeader.entrySet()); try { client.GET(sitemapURL.toString()); diff --git a/source/net/yacy/interaction/AugmentHtmlStream.java b/source/net/yacy/interaction/AugmentHtmlStream.java index 1861b9868..d0793c46a 100644 --- a/source/net/yacy/interaction/AugmentHtmlStream.java +++ b/source/net/yacy/interaction/AugmentHtmlStream.java @@ -9,6 +9,7 @@ import java.io.StringReader; import java.net.URLEncoder; import net.yacy.cora.document.ASCII; +import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.http.HTTPClient; @@ -31,7 +32,7 @@ public class AugmentHtmlStream { * @return the web page with integrated REFLECT elements */ private static String processExternal(String url, String fieldname, String data) throws IOException { - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); try { StringBuilder postdata = new StringBuilder(); postdata.append(fieldname); diff --git a/source/net/yacy/interaction/Interaction.java b/source/net/yacy/interaction/Interaction.java index e57f06761..0deba5d29 100644 --- a/source/net/yacy/interaction/Interaction.java +++ b/source/net/yacy/interaction/Interaction.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.Set; import net.yacy.cora.document.UTF8; +import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.http.HTTPClient; @@ -186,7 +187,7 @@ public static String Tableentry(String url, String type, String comment, String Seed host = sb.peers.lookupByName(sb.getConfig("interaction.contribution.accumulationpeer", "")); - return (UTF8.String(new HTTPClient().POSTbytes( + return (UTF8.String(new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT).POSTbytes( "http://"+host.getPublicAddress()+"/interaction/Contribution.json" + "?url=" + url + "&comment=" + comment + "&from=" + from + "&peer=" + peer, diff --git a/source/net/yacy/interaction/contentcontrol/ContentControlImportThread.java b/source/net/yacy/interaction/contentcontrol/ContentControlImportThread.java index d07d4c630..12173ae29 100644 --- a/source/net/yacy/interaction/contentcontrol/ContentControlImportThread.java +++ b/source/net/yacy/interaction/contentcontrol/ContentControlImportThread.java @@ -6,6 +6,7 @@ import java.net.MalformedURLException; import java.net.URL; import net.yacy.cora.document.UTF8; +import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.data.ymark.YMarkEntry; import net.yacy.data.ymark.YMarkSMWJSONImporter; @@ -15,47 +16,31 @@ import net.yacy.search.Switchboard; public class ContentControlImportThread { private final Switchboard sb; - private Boolean locked = false; - private String lastsync = "1900-01-01T01:00:00"; - private String currenttimestamp = "1900-01-01T01:00:00"; - private long offset = 0; - private final long limit = 500; - private long currentmax = 0; - private boolean runningjob = false; public ContentControlImportThread(final Switchboard sb) { - //final long time = System.currentTimeMillis(); - this.sb = sb; - if (this.sb.getConfigBool("contentcontrol.smwimport.purgelistoninit", - false)) { - this.sb.tables.clear(this.sb.getConfig( - "contentcontrol.smwimport.targetlist", "contentcontrol")); + if (this.sb.getConfigBool("contentcontrol.smwimport.purgelistoninit",false)) { + this.sb.tables.clear(this.sb.getConfig("contentcontrol.smwimport.targetlist", "contentcontrol")); } } private final String wikiurlify (String s) { - String ret = s; - ret = ret.replace("-", "-2D"); ret = ret.replace("+", "-2B"); ret = ret.replace(" ", "-20"); - ret = ret.replace("[", "-5B"); ret = ret.replace("]", "-5D"); - ret = ret.replace(":", "-3A"); ret = ret.replace(">", "-3E"); - ret = ret.replace("?", "-3F"); return ret; @@ -64,30 +49,17 @@ public class ContentControlImportThread { public final void run() { if (!this.locked) { - this.locked = true; - if (this.sb.getConfigBool("contentcontrol.smwimport.enabled", false) == true) { - if (this.runningjob) { - Log.logInfo("CONTENTCONTROL", "CONTENTCONTROL importing max. " + this.limit + " elements at " + this.offset + " of " + this.currentmax + ", since " + this.currenttimestamp); - URL bmks_json; - - //String currenttimestampurl = wikiurlify (this.currenttimestamp); - try { - - if (!this.sb.getConfig("contentcontrol.smwimport.baseurl", - "").equals("")) { - - - + if (!this.sb.getConfig("contentcontrol.smwimport.baseurl","").equals("")) { bmks_json = new URL( this.sb.getConfig( "contentcontrol.smwimport.baseurl", @@ -99,9 +71,7 @@ public class ContentControlImportThread { + "/offset%3D" + this.offset + "/limit%3D" + this.limit + "/format%3Djson"); - this.offset += this.limit; - if (this.offset > this.currentmax) { this.runningjob = false; } @@ -111,7 +81,6 @@ public class ContentControlImportThread { reader = new InputStreamReader( bmks_json.openStream(), "UTF-8"); } catch (Exception e) { - Log.logException(e); this.runningjob = false; } @@ -126,22 +95,14 @@ public class ContentControlImportThread { Log.logException(e); this.runningjob = false; } - Thread t; YMarkEntry bmk; - - t = new Thread(bookmarkImporter, - "YMarks - Network bookmark importer"); + t = new Thread(bookmarkImporter,"YMarks - Network bookmark importer"); t.start(); - while ((bmk = bookmarkImporter.take()) != YMarkEntry.POISON) { - if (bmk == YMarkEntry.EMPTY) { - this.runningjob = false; - } else { - try { this.sb.tables.bookmarks.addBookmark( this.sb.getConfig("contentcontrol.smwimport.targetlist", "contentcontrol"), bmk, @@ -153,12 +114,10 @@ public class ContentControlImportThread { } } } - } else { } } - else { } @@ -167,14 +126,9 @@ public class ContentControlImportThread { // TODO Auto-generated catch block e2.printStackTrace(); } - } else { - try { - - if (!this.sb.getConfig("contentcontrol.smwimport.baseurl", - "").equals("")) { - + if (!this.sb.getConfig("contentcontrol.smwimport.baseurl","").equals("")) { URL bmks_count; bmks_count = new URL( @@ -182,23 +136,16 @@ public class ContentControlImportThread { "contentcontrol.smwimport.baseurl", "") + wikiurlify ("/[[Category:Web Page]] [[Modification date::>" +this.lastsync+ "]]") - - + wikiurlify ("/?Url/?Filter/?Article has average rating/?Category") + "/mainlabel%3D" + "/format%3Dystat"); - String reply = UTF8.String(new HTTPClient() - .GETbytes(bmks_count.toString())); - + String reply = UTF8.String(new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT).GETbytes(bmks_count.toString())); String overallcount = reply.split(",")[0]; - String lastsyncstring = reply.split(",")[1]; - this.currentmax = Integer.parseInt(overallcount); if (this.currentmax > 0) { - Log.logInfo("CONTENTCONTROL", "CONTENTCONTROL import job counts " + this.currentmax @@ -216,7 +163,6 @@ public class ContentControlImportThread { Log.logWarning("CONTENTCONTROL", "No SMWimport URL defined"); } - } catch (MalformedURLException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -224,14 +170,10 @@ public class ContentControlImportThread { // TODO Auto-generated catch block e.printStackTrace(); } - } - this.locked = false; - } } - return; } diff --git a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java index aa97b5a70..369dadb0f 100644 --- a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java +++ b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java @@ -204,14 +204,8 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT // ratzen: German for to sleep (coll.) private void ratz(final long millis) { - try {/* - if (this.syncObject != null) { - synchronized (this.syncObject) { - this.syncObject.wait(millis); - } - } else {*/ - Thread.sleep(millis); - //} + try { + Thread.sleep(millis); } catch (final InterruptedException e) { if (log != null) log.logConfig("thread '" + this.getName() + "' interrupted because of shutdown."); diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 04d21f141..ece0b79c1 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -823,7 +823,7 @@ public final class SeedDB implements AlternativeDomainNames { reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary? reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent()); - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); client.setHeader(reqHeader.entrySet()); byte[] content = null; try { diff --git a/source/net/yacy/peers/operation/yacyRelease.java b/source/net/yacy/peers/operation/yacyRelease.java index 3aac1d55f..ae5a3e302 100644 --- a/source/net/yacy/peers/operation/yacyRelease.java +++ b/source/net/yacy/peers/operation/yacyRelease.java @@ -49,8 +49,6 @@ import net.yacy.cora.document.UTF8; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.storage.Files; @@ -288,16 +286,10 @@ public final class yacyRelease extends yacyVersion { public File downloadRelease() { final File storagePath = Switchboard.getSwitchboard().releasePath; File download = null; - // setup httpClient - final RequestHeader reqHeader = new RequestHeader(); - reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent()); final String name = getUrl().getFileName(); byte[] signatureBytes = null; - - final HTTPClient client = new HTTPClient(); - client.setTimout(6000); - client.setHeader(reqHeader.entrySet()); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); // download signature first, if public key is available try { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 6e405c738..6be9f24d2 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -3359,10 +3359,8 @@ public final class Switchboard extends serverSwitch final RequestHeader reqHeader = new RequestHeader(); reqHeader.put(HeaderFramework.PRAGMA, "no-cache"); reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache"); - reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent()); - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), timeout); client.setHeader(reqHeader.entrySet()); - client.setTimout(timeout); client.HEADResponse(url.toString()); int statusCode = client.getHttpResponse().getStatusLine().getStatusCode(); diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.java index 95bea1ec0..d15143d39 100644 --- a/source/net/yacy/server/http/HTTPDProxyHandler.java +++ b/source/net/yacy/server/http/HTTPDProxyHandler.java @@ -1074,8 +1074,7 @@ public final class HTTPDProxyHandler { */ private static HTTPClient setupHttpClient(final RequestHeader requestHeader, final String connectHost) { // setup HTTP-client - final HTTPClient client = new HTTPClient(); - client.setTimout(timeout); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), timeout); client.setHeader(requestHeader.entrySet()); client.setRedirecting(false); return client; diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index 05898b195..8cac6cdb4 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -610,7 +610,7 @@ public class serverSwitch try { final RequestHeader reqHeader = new RequestHeader(); reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent()); - final HTTPClient client = new HTTPClient(); + final HTTPClient client = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); client.setHeader(reqHeader.entrySet()); byte[] data = client.GETbytes(uri); if ( data == null || data.length == 0 ) { diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index a11f6ce96..7f3ba5547 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -531,7 +531,7 @@ public final class yacy { final RequestHeader requestHeader = new RequestHeader(); requestHeader.put(RequestHeader.AUTHORIZATION, "realm=" + encodedPassword); // for http-authentify // final Client con = new Client(10000, requestHeader); - final HTTPClient con = new HTTPClient(); + final HTTPClient con = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT); con.setHeader(requestHeader.entrySet()); // ResponseContainer res = null; try {