From 1488769e1f4733ad632e44baa7580ccb1bc3ba30 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 23 Sep 2007 20:49:52 +0000 Subject: [PATCH] cleanup of unmaintained and outdated performance methods: removed object pools in httpc. Object pooling is not recommended, if the creation of the object is not time-intensive. Object pools are only useful, if there is much computation necessary to create some basic data that is stored in the object pool and can be re-used. This does not apply to object pools in YaCy. Object pooling of client sessions would make sense if they would allow re-use of living connections to other yacy clients. But every connection is closed after usage of an object in the client pool, therefore the YaCy server client objects are not such that hold hardware/network-allocated entities. See: http://www.javaperformancetuning.com/news/qotm033.shtml http://java.sun.com/docs/hotspot/HotSpotFAQ.html#gc_pooling http://docs.sun.com/source/816-7159-10/pt_chap5.html http://www.microjava.com/articles/techtalk/recylcle2 git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4106 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/CrawlURLFetch_p.java | 5 +- source/de/anomic/data/SitemapParser.java | 6 +- source/de/anomic/data/robotsParser.java | 12 +- source/de/anomic/http/httpc.java | 432 +++--------------- source/de/anomic/http/httpdProxyHandler.java | 20 +- .../plasma/crawler/http/CrawlWorker.java | 12 +- source/de/anomic/plasma/plasmaCrawlLURL.java | 3 +- source/de/anomic/plasma/plasmaParser.java | 7 +- source/de/anomic/server/serverCore.java | 11 +- source/de/anomic/server/serverFileUtils.java | 18 +- source/de/anomic/yacy/yacySeedDB.java | 9 +- source/yacy.java | 2 +- 12 files changed, 109 insertions(+), 428 deletions(-) diff --git a/htroot/CrawlURLFetch_p.java b/htroot/CrawlURLFetch_p.java index 1488f4592..92ff415d5 100644 --- a/htroot/CrawlURLFetch_p.java +++ b/htroot/CrawlURLFetch_p.java @@ -507,13 +507,13 @@ public class CrawlURLFetch_p { if (url == null) return null; String[] r = null; try { - httpc con = httpc.getInstance( + httpc con = new httpc( url.getHost(), url.getHost(), url.getPort(), 15000, url.getProtocol().equals("https"), - plasmaSwitchboard.getSwitchboard().remoteProxyConfig); + plasmaSwitchboard.getSwitchboard().remoteProxyConfig, null, null); httpHeader header = new httpHeader(); header.put(httpHeader.ACCEPT_ENCODING, "US-ASCII"); @@ -531,7 +531,6 @@ public class CrawlURLFetch_p { if (encoding == null) encoding = "US-ASCII"; r = parseText(new String(sbb.getBytes(), encoding)); } - httpc.returnInstance(con); } catch (IOException e) { } return r; } diff --git a/source/de/anomic/data/SitemapParser.java b/source/de/anomic/data/SitemapParser.java index 45f421dea..3bec22e54 100644 --- a/source/de/anomic/data/SitemapParser.java +++ b/source/de/anomic/data/SitemapParser.java @@ -174,13 +174,13 @@ public class SitemapParser extends DefaultHandler { // download document httpc remote = null; try { - remote = httpc.getInstance( + remote = new httpc( this.siteMapURL.getHost(), this.siteMapURL.getHost(), this.siteMapURL.getPort(), 5000, this.siteMapURL.getProtocol().equalsIgnoreCase("https"), - switchboard.remoteProxyConfig); + switchboard.remoteProxyConfig, null, null); httpc.response res = remote.GET(this.siteMapURL.getFile(), null); if (res.statusCode != 200) { @@ -211,8 +211,6 @@ public class SitemapParser extends DefaultHandler { saxParser.parse(this.counterStream, this); } catch (Exception e) { this.logger.logWarning("Unable to parse sitemap file " + this.siteMapURL,e); - } finally { - if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {/* ignore this */} } } diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java index e33b006ea..f731b4243 100644 --- a/source/de/anomic/data/robotsParser.java +++ b/source/de/anomic/data/robotsParser.java @@ -386,7 +386,7 @@ public final class robotsParser{ downloadStart = System.currentTimeMillis(); plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard(); //TODO: adding Traffic statistic for robots download? - con = httpc.getInstance(robotsURL.getHost(), robotsURL.getHost(), robotsURL.getPort(), 10000, robotsURL.getProtocol().equalsIgnoreCase("https"), sb.remoteProxyConfig); + con = new httpc(robotsURL.getHost(), robotsURL.getHost(), robotsURL.getPort(), 10000, robotsURL.getProtocol().equalsIgnoreCase("https"), sb.remoteProxyConfig, null, null); // if we previously have downloaded this robots.txt then we can set the if-modified-since header httpHeader reqHeaders = new httpHeader(); @@ -447,11 +447,7 @@ public final class robotsParser{ redirectionUrlString = redirectionUrlString.trim(); // generating the new URL object - yacyURL redirectionUrl = yacyURL.newURL(robotsURL, redirectionUrlString); - - // returning the used httpc - httpc.returnInstance(con); - con = null; + yacyURL redirectionUrl = yacyURL.newURL(robotsURL, redirectionUrlString); // following the redirection serverLog.logFinest("ROBOTS","Redirection detected for robots.txt with URL '" + robotsURL + "'." + @@ -467,9 +463,7 @@ public final class robotsParser{ } } catch (Exception e) { throw e; - } finally { - if (con != null) httpc.returnInstance(con); - } + } return new Object[]{new Boolean(accessCompletelyRestricted),robotsTxt,eTag,lastMod}; } } diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index ec2e6d88e..f98ae8f06 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -74,8 +74,6 @@ import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; -import org.apache.commons.pool.impl.GenericObjectPool; - import de.anomic.kelondro.kelondroBase64Order; import de.anomic.server.serverByteBuffer; import de.anomic.server.serverCore; @@ -91,8 +89,8 @@ import de.anomic.yacy.yacyURL; * libraries, it is still necessary to implement the network interface since * otherwise there is no access to the HTTP/1.0 / HTTP/1.1 header information * that comes along each connection. -* FIXME: Add some information about the usage of the threadpool. */ + public final class httpc { // some constants @@ -103,87 +101,30 @@ public final class httpc { */ public static final String GZIP_POST_BODY = "GZIP_POST_BODY"; - // statics + // final statics private static final String vDATE = "20040602"; - public static String userAgent; private static final int terminalMaxLength = 30000; private static final TimeZone GMTTimeZone = TimeZone.getTimeZone("GMT"); - - /** - * This string is initialized on loading of this class and contains - * information about the current OS. - */ - public static String systemOST; - - // --- The GMT standard date format used in the HTTP protocol private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US); - static { - HTTPGMTFormatter.setTimeZone(GMTTimeZone); - } - - /** - * A Object Pool containing all pooled httpc-objects. - * @see httpcPool - */ - private static final httpcPool theHttpcPool; - - // class variables - private Socket socket = null; // client socket for commands - private Thread socketOwner = null; - private String adressed_host = null; - private int adressed_port = 80; - private String target_virtual_host = null; - - // output and input streams for client control connection - PushbackInputStream clientInput = null; - private OutputStream clientOutput = null; - - private httpdByteCountInputStream clientInputByteCount = null; - private httpdByteCountOutputStream clientOutputByteCount = null; - - private boolean remoteProxyUse = false; - private httpRemoteProxyConfig remoteProxyConfig = null; + private static final HashMap reverseMappingCache = new HashMap(); + private static final HashMap openSocketLookupTable = new HashMap(); - String requestPath = null; - private boolean allowContentEncoding = true; - public static boolean yacyDebugMode = false; - - static final HashMap reverseMappingCache = new HashMap(); + // defined during set-up of switchboard + public static boolean yacyDebugMode = false; - /** - * Indicates if the current object was removed from pool because the maximum limit - * was exceeded. - */ - boolean removedFromPool = false; - - static SSLSocketFactory theSSLSockFactory = null; + // statics to be defined in static section below + private static SSLSocketFactory theSSLSockFactory = null; + public static String systemOST; + public static String userAgent; static { + // set the time zone + HTTPGMTFormatter.setTimeZone(GMTTimeZone); // The GMT standard date format used in the HTTP protocol + // set time-out of InetAddress.getByName cache ttl java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0"); - - // Configuring the httpc object pool - - // implementation of session thread pool - GenericObjectPool.Config config = new GenericObjectPool.Config(); - - // The maximum number of active connections that can be allocated from pool at the same time, - // 0 for no limit - config.maxActive = 150; - - // The maximum number of idle connections connections in the pool - // 0 = no limit. - config.maxIdle = 75; - config.minIdle = 10; - - config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; - config.minEvictableIdleTimeMillis = 30000; - - theHttpcPool = new httpcPool(new httpcFactory(),config); - - // initializing a dummy trustManager to enable https connections // Create a trust manager that does not validate certificate chains @@ -218,8 +159,7 @@ public final class httpc { HttpsURLConnection.setDefaultHostnameVerifier(hv); } catch (Exception e) { } - - + // provide system information for client identification String loc = System.getProperty("user.timezone", "nowhere"); int p = loc.indexOf("/"); @@ -233,13 +173,27 @@ public final class httpc { userAgent = "yacy (www.yacy.net; v" + vDATE + "; " + systemOST + ")"; } - /** - * A reusable readline buffer - * @see serverByteBuffer - */ - final serverByteBuffer readLineBuffer = new serverByteBuffer(100); + + // class variables + private Socket socket = null; // client socket for commands + private Thread socketOwner = null; + private String adressed_host = null; + private int adressed_port = 80; + private String target_virtual_host = null; + + // output and input streams for client control connection + private PushbackInputStream clientInput = null; + private OutputStream clientOutput = null; + + private httpdByteCountInputStream clientInputByteCount = null; + private httpdByteCountOutputStream clientOutputByteCount = null; - private static final HashMap openSocketLookupTable = new HashMap(); + private boolean remoteProxyUse = false; + private httpRemoteProxyConfig remoteProxyConfig = null; + + private String requestPath = null; + private boolean allowContentEncoding = true; + /** * Convert the status of this class into an String object to output it. @@ -249,83 +203,6 @@ public final class httpc { ((this.remoteProxyUse) ? " via " + adressed_host : ""); } - /** - * This method gets a new httpc instance from the object pool and - * initializes it with the given parameters. Use this method if you have to - * use a proxy to access the pages. - * - * @param server - * @param port - * @param timeout - * @param ssl - * @param remoteProxyHost - * @param remoteProxyPort - * @throws IOException - * @see httpc#init - */ - public static httpc getInstance( - String server, - String vhost, - int port, - int timeout, - boolean ssl, - httpRemoteProxyConfig remoteProxyConfig, - String incomingByteCountAccounting, - String outgoingByteCountAccounting - ) throws IOException { - - httpc newHttpc; - try { - // fetching a new httpc from the object pool - newHttpc = (httpc) httpc.theHttpcPool.borrowObject(); - } catch (Exception e) { - throw new IOException("Unable to initialize a new httpc. " + e.getMessage()); - } - - // initialize it - try { - newHttpc.init( - server, - vhost, - port, - timeout, - ssl, - remoteProxyConfig, - incomingByteCountAccounting, - outgoingByteCountAccounting - ); - } catch (IOException e) { - try{ httpc.theHttpcPool.returnObject(newHttpc); } catch (Exception e1) {} - throw e; - } - return newHttpc; - } - - public static httpc getInstance( - String server, - String vhost, - int port, - int timeout, - boolean ssl, - httpRemoteProxyConfig remoteProxyConfig - ) throws IOException { - return getInstance(server,vhost,port,timeout,ssl,remoteProxyConfig,null,null); - } - - /** - * Put back a used instance into the instance pool of httpc. - * - * @param theHttpc The instance of httpc which should be returned to the pool - */ - public static void returnInstance(httpc theHttpc) { - try { - theHttpc.reset(); - httpc.theHttpcPool.returnObject(theHttpc); - } catch (Exception e) { - // we could ignore this error - } - } - /** * Sets wether the content is allowed to be unzipped while getting? * FIXME: The name of this method seems misleading, if I read the usage of @@ -377,15 +254,13 @@ public final class httpc { } /** - * Initialize the httpc-instance with the given data. This method is used, - * if you have to use a proxy to access the pages. This just calls init - * without proxy information and adds the proxy information. + * Initialize the httpc-instance with the given data. * * @param remoteProxyHost * @param remoteProxyPort * @throws IOException */ - private void init( + public httpc( String server, String vhost, int port, @@ -506,9 +381,6 @@ public final class httpc { this.socket.getInputStream()); this.clientOutput = this.socket.getOutputStream(); - - - // if we reached this point, we should have a connection } catch (UnknownHostException e) { if (this.socket != null) { @@ -528,69 +400,7 @@ public final class httpc { return (this.clientOutputByteCount == null)?0:this.clientOutputByteCount.getCount(); } - /** - * This method resets an httpc-instance, so that it can be used for the next - * connection. This is called before the instance is put back to the pool. - * All streams and sockets are closed and set to null. - */ - void reset() { - if (this.clientInput != null) { - try {this.clientInput.close();} catch (Exception e) {} - this.clientInput = null; - } - if (this.clientOutput != null) { - try {this.clientOutput.close();} catch (Exception e) {} - this.clientOutput = null; - } - if (this.socket != null) { - try {this.socket.close();} catch (Exception e) {} - httpc.unregisterOpenSocket(this.socket,this.socketOwner); - this.socket = null; - this.socketOwner = null; - } - - if (this.clientInputByteCount != null) { - this.clientInputByteCount.finish(); - this.clientInputByteCount = null; - } - if (this.clientOutputByteCount != null) { - this.clientOutputByteCount.finish(); - this.clientOutputByteCount = null; - } - - this.adressed_host = null; - this.target_virtual_host = null; - //this.timeout = 0; - - this.remoteProxyUse = false; - this.remoteProxyConfig = null; - this.requestPath = null; - - this.allowContentEncoding = true; - - // shrink readlinebuffer if it is to large - this.readLineBuffer.reset(80); - } - - /** - * Just calls reset to close all connections. - */ public void close() { - reset(); - } - - /** - * If this instance is garbage-collected we check if the object was returned - * to the pool. if not, we invalidate the object from the pool. - * - * @see httpcPool#invalidateObject - */ - protected void finalize() throws Throwable { - if (!(this.removedFromPool)) { - System.err.println("Httpc object was not returned to object pool."); - httpc.theHttpcPool.invalidateObject(this); - } - this.reset(); } /** @@ -939,29 +749,22 @@ public final class httpc { } httpc con = null; - try { - con = httpc.getInstance(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig); - - httpc.response res = con.GET(path, requestHeader); - if (res.status.startsWith("2")) { - if (download == null) { - // stream to byte[] - serverByteBuffer sbb = new serverByteBuffer(); - res.writeContent(sbb, null); - return sbb.getBytes(); - } else { - // stream to file and return null - res.writeContent(null, download); - return null; - } + con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null); + + httpc.response res = con.GET(path, requestHeader); + if (res.status.startsWith("2")) { + if (download == null) { + // stream to byte[] + serverByteBuffer sbb = new serverByteBuffer(); + res.writeContent(sbb, null); + return sbb.getBytes(); + } else { + // stream to file and return null + res.writeContent(null, download); + return null; } - return res.status.getBytes(); - } catch (Exception e) { - throw new IOException(e.getMessage()); - } finally { - if (con != null) httpc.returnInstance(con); } - + return res.status.getBytes(); } public static byte[] singleGET( @@ -1002,24 +805,16 @@ public final class httpc { requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password)); } - httpc con = null; - try { - con = httpc.getInstance(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig); - httpc.response res = con.POST(path, requestHeader, props, files); - - //System.out.println("response=" + res.toString()); - if (res.status.startsWith("2")) { - serverByteBuffer sbb = new serverByteBuffer(); - res.writeContent(sbb, null); - return sbb.getBytes(); - } - return res.status.getBytes(); - } catch (Exception e) { - throw new IOException(e.getMessage()); - } finally { - if (con != null) httpc.returnInstance(con); - } + httpc con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null); + httpc.response res = con.POST(path, requestHeader, props, files); + //System.out.println("response=" + res.toString()); + if (!(res.status.startsWith("2"))) return res.status.getBytes(); + + // read connection body and return body + serverByteBuffer sbb = new serverByteBuffer(); + res.writeContent(sbb, null); + return sbb.getBytes(); } public static byte[] singlePOST( @@ -1161,20 +956,14 @@ public final class httpc { // start connection httpc con = null; - try { - con = httpc.getInstance(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig); - httpc.response res = con.HEAD(path, requestHeader); - if (res.status.startsWith("2")) { - // success - return res.responseHeader; - } - // fail + con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null); + httpc.response res = con.HEAD(path, requestHeader); + if (res.status.startsWith("2")) { + // success return res.responseHeader; - } catch (Exception e) { - throw new IOException(e.getMessage()); - } finally { - if (con != null) httpc.returnInstance(con); } + // fail + return res.responseHeader; } public static byte[] wput( @@ -1456,7 +1245,7 @@ public final class httpc { } // reads in the http header, right now, right here - byte[] b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false); + byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false); if (b == null) { // the server has meanwhile disconnected this.statusCode = 503; @@ -1475,7 +1264,7 @@ public final class httpc { if ((this.statusCode==500)&&(this.statusText.equals("status line parse error"))) { // flush in anything that comes without parsing - while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false); + while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false); return; // in bad mood } @@ -1483,13 +1272,13 @@ public final class httpc { if (this.statusCode == 400) { // bad request // flush in anything that comes without parsing - while ((b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false)).length != 0) {} + while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)).length != 0) {} return; // in bad mood } // at this point we should have a valid response. read in the header properties String key = ""; - while ((b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false)) != null) { + while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)) != null) { if (b.length == 0) break; buffer = new String(b); buffer=buffer.trim(); @@ -1786,86 +1575,3 @@ public class SSLSocketClientWithClientAuth { } } */ - -final class httpcFactory implements org.apache.commons.pool.PoolableObjectFactory { - - public httpcFactory() { - super(); - } - - /** - * @see org.apache.commons.pool.PoolableObjectFactory#makeObject() - */ - public Object makeObject() throws Exception { - return new httpc(); - } - - /** - * @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object) - */ - public void destroyObject(Object obj) { - assert(obj instanceof httpc): "Invalid object type added to pool."; - if (obj instanceof httpc) { - httpc theHttpc = (httpc) obj; - - theHttpc.removedFromPool = true; - } - } - - /** - * @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object) - */ - public boolean validateObject(Object obj) { - assert(obj instanceof httpc): "Invalid object type in pool."; - return true; - } - - /** - * @param obj - * - */ - public void activateObject(Object obj) { - //log.debug(" activateObject..."); - } - - /** - * @param obj - * - */ - public void passivateObject(Object obj) { - assert(obj instanceof httpc): "Invalid object type returned to pool."; - } -} - -final class httpcPool extends GenericObjectPool { - /** - * First constructor. - * @param objFactory - */ - public httpcPool(httpcFactory objFactory) { - super(objFactory); - this.setMaxIdle(75); // Maximum idle threads. - this.setMaxActive(150); // Maximum active threads. - this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs. - //this.setMaxWait(1000); // Wait 1 second till a thread is available - } - - public httpcPool(httpcFactory objFactory, - GenericObjectPool.Config config) { - super(objFactory, config); - } - - /** - * @see org.apache.commons.pool.impl.GenericObjectPool#borrowObject() - */ - public Object borrowObject() throws Exception { - return super.borrowObject(); - } - - /** - * @see org.apache.commons.pool.impl.GenericObjectPool#returnObject(java.lang.Object) - */ - public void returnObject(Object obj) throws Exception { - super.returnObject(obj); - } -} diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 5faedd672..50711a27b 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -719,9 +719,7 @@ public final class httpdProxyHandler { // deleting cached content if (cacheFile.exists()) cacheFile.delete(); handleProxyException(e,remote,conProp,respond,url); - } finally { - if (remote != null) httpc.returnInstance(remote); - } + } } @@ -958,8 +956,6 @@ public final class httpdProxyHandler { httpd.sendRespondHeader(conProp,respond,httpVer,res.statusCode,res.statusText,res.responseHeader); } catch (Exception e) { handleProxyException(e,remote,conProp,respond,url); - } finally { - if (remote != null) httpc.returnInstance(remote); } respond.flush(); @@ -1085,8 +1081,6 @@ public final class httpdProxyHandler { } catch (Exception e) { handleProxyException(e,remote,conProp,respond,url); } finally { - if (remote != null) httpc.returnInstance(remote); - respond.flush(); if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish(); @@ -1134,13 +1128,14 @@ public final class httpdProxyHandler { ) { httpc remoteProxy = null; try { - remoteProxy = httpc.getInstance( + remoteProxy = new httpc( host, host, port, timeout, false, - switchboard.remoteProxyConfig + switchboard.remoteProxyConfig, + null, null ); httpc.response response = remoteProxy.CONNECT(host, port, requestHeader); @@ -1159,8 +1154,6 @@ public final class httpdProxyHandler { } } catch (Exception e) { throw new IOException(e.getMessage()); - } finally { - if (remoteProxy != null) httpc.returnInstance(remoteProxy); } } @@ -1271,13 +1264,14 @@ public final class httpdProxyHandler { } // branch to server/proxy - return httpc.getInstance( + return new httpc( server, server, port, timeout, false, - remProxyConfig + remProxyConfig, + null, null ); } diff --git a/source/de/anomic/plasma/crawler/http/CrawlWorker.java b/source/de/anomic/plasma/crawler/http/CrawlWorker.java index 329d06da1..3fb4a9b0d 100644 --- a/source/de/anomic/plasma/crawler/http/CrawlWorker.java +++ b/source/de/anomic/plasma/crawler/http/CrawlWorker.java @@ -193,7 +193,7 @@ public final class CrawlWorker extends AbstractCrawlWorker { requestHeader.put(httpHeader.ACCEPT_ENCODING, this.acceptEncoding); // open the connection - remote = httpc.getInstance(host, host, port, this.socketTimeout, ssl, this.remoteProxyConfig,"CRAWLER",null); + remote = new httpc(host, host, port, this.socketTimeout, ssl, this.remoteProxyConfig, "CRAWLER", null); // specifying if content encoding is allowed remote.setAllowContentEncoding((this.acceptEncoding != null && this.acceptEncoding.length() > 0)); @@ -315,10 +315,6 @@ public final class CrawlWorker extends AbstractCrawlWorker { // normalizing URL yacyURL redirectionUrl = yacyURL.newURL(this.url, redirectionUrlString); - // returning the used httpc - httpc.returnInstance(remote); - remote = null; - // restart crawling with new url this.log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + this.url.toString()); this.log.logInfo("CRAWLER ..Redirecting request to: " + redirectionUrl); @@ -452,10 +448,6 @@ public final class CrawlWorker extends AbstractCrawlWorker { return null; } - // returning the used httpc - if (remote != null) httpc.returnInstance(remote); - remote = null; - // setting the retry counter to 1 if (crawlingRetryCount > 2) crawlingRetryCount = 2; @@ -467,8 +459,6 @@ public final class CrawlWorker extends AbstractCrawlWorker { addURLtoErrorDB(failreason); } return null; - } finally { - if (remote != null) httpc.returnInstance(remote); } } diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 055f6fe4c..7620cc745 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -398,7 +398,7 @@ public final class plasmaCrawlLURL { yacyURL newUrl = new yacyURL(newUrlStr, null); // doing a http head request to test if the url is correct - theHttpc = httpc.getInstance(newUrl.getHost(), newUrl.getHost(), newUrl.getPort(), 30000, false, plasmaSwitchboard.getSwitchboard().remoteProxyConfig); + theHttpc = new httpc(newUrl.getHost(), newUrl.getHost(), newUrl.getPort(), 30000, false, plasmaSwitchboard.getSwitchboard().remoteProxyConfig, null, null); response res = theHttpc.HEAD(newUrl.getPath(), null); if (res.statusCode == 200) { @@ -416,7 +416,6 @@ public final class plasmaCrawlLURL { } finally { if (theHttpc != null) try { theHttpc.close(); - httpc.returnInstance(theHttpc); } catch (Exception e) { } } } diff --git a/source/de/anomic/plasma/plasmaParser.java b/source/de/anomic/plasma/plasmaParser.java index 2d7e165e1..502cbe879 100644 --- a/source/de/anomic/plasma/plasmaParser.java +++ b/source/de/anomic/plasma/plasmaParser.java @@ -914,13 +914,14 @@ public final class plasmaParser { contentURL = new yacyURL(args[1], null); // downloading the document content - remote = httpc.getInstance( + remote = new httpc( contentURL.getHost(), contentURL.getHost(), contentURL.getPort(), 5000, contentURL.getProtocol().equalsIgnoreCase("https"), - null); + null, + null, null); httpc.response res = remote.GET(contentURL.getFile(), null); if (res.statusCode != 200) { @@ -987,8 +988,6 @@ public final class plasmaParser { } } catch (Exception e) { e.printStackTrace(); - } finally { - if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {} } } diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 37084a213..effdf4886 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -117,7 +117,6 @@ public final class serverCore extends serverAbstractThread implements serverThre private ServerSocket socket; // listener serverLog log; // log object private int timeout; // connection time-out of the socket - private int thresholdSleep = 30000; // after that time a thread is considered as beeing sleeping (30 seconds) serverHandler handlerPrototype; // the command class (a serverHandler) @@ -762,9 +761,6 @@ public final class serverCore extends serverAbstractThread implements serverThre public OutputStream out; // on control output stream, autoflush public int socketTimeout; - private final serverByteBuffer readLineBuffer = new serverByteBuffer(256); - - public Session(ThreadGroup theThreadGroup) { super(theThreadGroup,"Session_created"); } @@ -864,7 +860,7 @@ public final class serverCore extends serverAbstractThread implements serverThre } public byte[] readLine() { - return receive(this.in, this.readLineBuffer, serverCore.this.commandMaxLength, false); + return receive(this.in, serverCore.this.commandMaxLength, false); } /** @@ -890,7 +886,6 @@ public final class serverCore extends serverAbstractThread implements serverThre public void reset() { this.done = true; this.syncObject = null; - this.readLineBuffer.reset(); if (this.commandObj !=null) this.commandObj.reset(); this.userAddress = null; this.userPort = 0; @@ -1174,10 +1169,10 @@ public final class serverCore extends serverAbstractThread implements serverThre } - public static byte[] receive(PushbackInputStream pbis, serverByteBuffer readLineBuffer, int maxSize, boolean logerr) { + public static byte[] receive(PushbackInputStream pbis, int maxSize, boolean logerr) { // reuse an existing linebuffer - readLineBuffer.reset(); + serverByteBuffer readLineBuffer = new serverByteBuffer(80); int bufferSize = 0, b = 0; try { diff --git a/source/de/anomic/server/serverFileUtils.java b/source/de/anomic/server/serverFileUtils.java index 6e3791cc2..4b2e7b511 100644 --- a/source/de/anomic/server/serverFileUtils.java +++ b/source/de/anomic/server/serverFileUtils.java @@ -113,14 +113,18 @@ public final class serverFileUtils { public static void writeX(InputStream source, OutputStream procOS, OutputStream bufferOS) { byte[] buffer = new byte[2048]; - int l; - + int l, c = 0; + while (true) try { l = source.read(buffer, 0, buffer.length); if (l <= 0) break; + c += l; if (procOS != null) procOS.write(buffer, 0, l); if (bufferOS != null) bufferOS.write(buffer, 0, l); - } catch (IOException e) {break;} + } catch (IOException e) { + System.out.println("** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes."); + break; + } // flush the streams if (procOS != null) try { procOS.flush(); } catch (IOException e) {} @@ -130,14 +134,18 @@ public final class serverFileUtils { public static void writeX(Reader source, Writer procOS, Writer bufferOS) { char[] buffer = new char[2048]; - int l; + int l, c= 0; while (true) try{ l = source.read(buffer, 0, buffer.length); if (l <= 0) break; + c += l; if (procOS != null) procOS.write(buffer, 0, l); if (bufferOS != null) bufferOS.write(buffer, 0, l); - } catch (IOException e) {break;} + } catch (IOException e) { + System.out.println("** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes."); + break; + } // flush the streams if (procOS != null) try { procOS.flush(); } catch (IOException e) {} diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 1341d41b7..bfd7a8f88 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -769,13 +769,14 @@ public final class yacySeedDB { httpc remote = null; try { // init httpc - remote = httpc.getInstance( + remote = new httpc( seedURL.getHost(), seedURL.getHost(), seedURL.getPort(), 10000, seedURL.getProtocol().equalsIgnoreCase("https"), - sb.remoteProxyConfig); + sb.remoteProxyConfig, + null, null); // Configure http headers httpHeader reqHeader = new httpHeader(); @@ -801,9 +802,7 @@ public final class yacySeedDB { return nxTools.strings(content,"UTF-8"); } catch (Exception e) { throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage()); - } finally { - if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {} - } + } } private String checkCache(ArrayList uv, ArrayList check) { diff --git a/source/yacy.java b/source/yacy.java index d4df71164..67dcb4e5c 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -509,7 +509,7 @@ public final class yacy { httpHeader requestHeader = new httpHeader(); requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify try { - httpc con = httpc.getInstance("localhost", "localhost", port, 10000, false, null); + httpc con = new httpc("localhost", "localhost", port, 10000, false, null, null, null); httpc.response res = con.GET("Steering.html?shutdown=", requestHeader); // read response