diff --git a/htroot/CrawlURLFetch_p.java b/htroot/CrawlURLFetch_p.java index 1488f4592..92ff415d5 100644 --- a/htroot/CrawlURLFetch_p.java +++ b/htroot/CrawlURLFetch_p.java @@ -507,13 +507,13 @@ public class CrawlURLFetch_p { if (url == null) return null; String[] r = null; try { - httpc con = httpc.getInstance( + httpc con = new httpc( url.getHost(), url.getHost(), url.getPort(), 15000, url.getProtocol().equals("https"), - plasmaSwitchboard.getSwitchboard().remoteProxyConfig); + plasmaSwitchboard.getSwitchboard().remoteProxyConfig, null, null); httpHeader header = new httpHeader(); header.put(httpHeader.ACCEPT_ENCODING, "US-ASCII"); @@ -531,7 +531,6 @@ public class CrawlURLFetch_p { if (encoding == null) encoding = "US-ASCII"; r = parseText(new String(sbb.getBytes(), encoding)); } - httpc.returnInstance(con); } catch (IOException e) { } return r; } diff --git a/source/de/anomic/data/SitemapParser.java b/source/de/anomic/data/SitemapParser.java index 45f421dea..3bec22e54 100644 --- a/source/de/anomic/data/SitemapParser.java +++ b/source/de/anomic/data/SitemapParser.java @@ -174,13 +174,13 @@ public class SitemapParser extends DefaultHandler { // download document httpc remote = null; try { - remote = httpc.getInstance( + remote = new httpc( this.siteMapURL.getHost(), this.siteMapURL.getHost(), this.siteMapURL.getPort(), 5000, this.siteMapURL.getProtocol().equalsIgnoreCase("https"), - switchboard.remoteProxyConfig); + switchboard.remoteProxyConfig, null, null); httpc.response res = remote.GET(this.siteMapURL.getFile(), null); if (res.statusCode != 200) { @@ -211,8 +211,6 @@ public class SitemapParser extends DefaultHandler { saxParser.parse(this.counterStream, this); } catch (Exception e) { this.logger.logWarning("Unable to parse sitemap file " + this.siteMapURL,e); - } finally { - if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {/* ignore this */} } } diff --git a/source/de/anomic/data/robotsParser.java b/source/de/anomic/data/robotsParser.java index e33b006ea..f731b4243 100644 --- a/source/de/anomic/data/robotsParser.java +++ b/source/de/anomic/data/robotsParser.java @@ -386,7 +386,7 @@ public final class robotsParser{ downloadStart = System.currentTimeMillis(); plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard(); //TODO: adding Traffic statistic for robots download? - con = httpc.getInstance(robotsURL.getHost(), robotsURL.getHost(), robotsURL.getPort(), 10000, robotsURL.getProtocol().equalsIgnoreCase("https"), sb.remoteProxyConfig); + con = new httpc(robotsURL.getHost(), robotsURL.getHost(), robotsURL.getPort(), 10000, robotsURL.getProtocol().equalsIgnoreCase("https"), sb.remoteProxyConfig, null, null); // if we previously have downloaded this robots.txt then we can set the if-modified-since header httpHeader reqHeaders = new httpHeader(); @@ -447,11 +447,7 @@ public final class robotsParser{ redirectionUrlString = redirectionUrlString.trim(); // generating the new URL object - yacyURL redirectionUrl = yacyURL.newURL(robotsURL, redirectionUrlString); - - // returning the used httpc - httpc.returnInstance(con); - con = null; + yacyURL redirectionUrl = yacyURL.newURL(robotsURL, redirectionUrlString); // following the redirection serverLog.logFinest("ROBOTS","Redirection detected for robots.txt with URL '" + robotsURL + "'." + @@ -467,9 +463,7 @@ public final class robotsParser{ } } catch (Exception e) { throw e; - } finally { - if (con != null) httpc.returnInstance(con); - } + } return new Object[]{new Boolean(accessCompletelyRestricted),robotsTxt,eTag,lastMod}; } } diff --git a/source/de/anomic/http/httpc.java b/source/de/anomic/http/httpc.java index ec2e6d88e..f98ae8f06 100644 --- a/source/de/anomic/http/httpc.java +++ b/source/de/anomic/http/httpc.java @@ -74,8 +74,6 @@ import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; -import org.apache.commons.pool.impl.GenericObjectPool; - import de.anomic.kelondro.kelondroBase64Order; import de.anomic.server.serverByteBuffer; import de.anomic.server.serverCore; @@ -91,8 +89,8 @@ import de.anomic.yacy.yacyURL; * libraries, it is still necessary to implement the network interface since * otherwise there is no access to the HTTP/1.0 / HTTP/1.1 header information * that comes along each connection. -* FIXME: Add some information about the usage of the threadpool. */ + public final class httpc { // some constants @@ -103,87 +101,30 @@ public final class httpc { */ public static final String GZIP_POST_BODY = "GZIP_POST_BODY"; - // statics + // final statics private static final String vDATE = "20040602"; - public static String userAgent; private static final int terminalMaxLength = 30000; private static final TimeZone GMTTimeZone = TimeZone.getTimeZone("GMT"); - - /** - * This string is initialized on loading of this class and contains - * information about the current OS. - */ - public static String systemOST; - - // --- The GMT standard date format used in the HTTP protocol private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US); - static { - HTTPGMTFormatter.setTimeZone(GMTTimeZone); - } - - /** - * A Object Pool containing all pooled httpc-objects. - * @see httpcPool - */ - private static final httpcPool theHttpcPool; - - // class variables - private Socket socket = null; // client socket for commands - private Thread socketOwner = null; - private String adressed_host = null; - private int adressed_port = 80; - private String target_virtual_host = null; - - // output and input streams for client control connection - PushbackInputStream clientInput = null; - private OutputStream clientOutput = null; - - private httpdByteCountInputStream clientInputByteCount = null; - private httpdByteCountOutputStream clientOutputByteCount = null; - - private boolean remoteProxyUse = false; - private httpRemoteProxyConfig remoteProxyConfig = null; + private static final HashMap reverseMappingCache = new HashMap(); + private static final HashMap openSocketLookupTable = new HashMap(); - String requestPath = null; - private boolean allowContentEncoding = true; - public static boolean yacyDebugMode = false; - - static final HashMap reverseMappingCache = new HashMap(); + // defined during set-up of switchboard + public static boolean yacyDebugMode = false; - /** - * Indicates if the current object was removed from pool because the maximum limit - * was exceeded. - */ - boolean removedFromPool = false; - - static SSLSocketFactory theSSLSockFactory = null; + // statics to be defined in static section below + private static SSLSocketFactory theSSLSockFactory = null; + public static String systemOST; + public static String userAgent; static { + // set the time zone + HTTPGMTFormatter.setTimeZone(GMTTimeZone); // The GMT standard date format used in the HTTP protocol + // set time-out of InetAddress.getByName cache ttl java.security.Security.setProperty("networkaddress.cache.ttl" , "60"); java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0"); - - // Configuring the httpc object pool - - // implementation of session thread pool - GenericObjectPool.Config config = new GenericObjectPool.Config(); - - // The maximum number of active connections that can be allocated from pool at the same time, - // 0 for no limit - config.maxActive = 150; - - // The maximum number of idle connections connections in the pool - // 0 = no limit. - config.maxIdle = 75; - config.minIdle = 10; - - config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK; - config.minEvictableIdleTimeMillis = 30000; - - theHttpcPool = new httpcPool(new httpcFactory(),config); - - // initializing a dummy trustManager to enable https connections // Create a trust manager that does not validate certificate chains @@ -218,8 +159,7 @@ public final class httpc { HttpsURLConnection.setDefaultHostnameVerifier(hv); } catch (Exception e) { } - - + // provide system information for client identification String loc = System.getProperty("user.timezone", "nowhere"); int p = loc.indexOf("/"); @@ -233,13 +173,27 @@ public final class httpc { userAgent = "yacy (www.yacy.net; v" + vDATE + "; " + systemOST + ")"; } - /** - * A reusable readline buffer - * @see serverByteBuffer - */ - final serverByteBuffer readLineBuffer = new serverByteBuffer(100); + + // class variables + private Socket socket = null; // client socket for commands + private Thread socketOwner = null; + private String adressed_host = null; + private int adressed_port = 80; + private String target_virtual_host = null; + + // output and input streams for client control connection + private PushbackInputStream clientInput = null; + private OutputStream clientOutput = null; + + private httpdByteCountInputStream clientInputByteCount = null; + private httpdByteCountOutputStream clientOutputByteCount = null; - private static final HashMap openSocketLookupTable = new HashMap(); + private boolean remoteProxyUse = false; + private httpRemoteProxyConfig remoteProxyConfig = null; + + private String requestPath = null; + private boolean allowContentEncoding = true; + /** * Convert the status of this class into an String object to output it. @@ -249,83 +203,6 @@ public final class httpc { ((this.remoteProxyUse) ? " via " + adressed_host : ""); } - /** - * This method gets a new httpc instance from the object pool and - * initializes it with the given parameters. Use this method if you have to - * use a proxy to access the pages. - * - * @param server - * @param port - * @param timeout - * @param ssl - * @param remoteProxyHost - * @param remoteProxyPort - * @throws IOException - * @see httpc#init - */ - public static httpc getInstance( - String server, - String vhost, - int port, - int timeout, - boolean ssl, - httpRemoteProxyConfig remoteProxyConfig, - String incomingByteCountAccounting, - String outgoingByteCountAccounting - ) throws IOException { - - httpc newHttpc; - try { - // fetching a new httpc from the object pool - newHttpc = (httpc) httpc.theHttpcPool.borrowObject(); - } catch (Exception e) { - throw new IOException("Unable to initialize a new httpc. " + e.getMessage()); - } - - // initialize it - try { - newHttpc.init( - server, - vhost, - port, - timeout, - ssl, - remoteProxyConfig, - incomingByteCountAccounting, - outgoingByteCountAccounting - ); - } catch (IOException e) { - try{ httpc.theHttpcPool.returnObject(newHttpc); } catch (Exception e1) {} - throw e; - } - return newHttpc; - } - - public static httpc getInstance( - String server, - String vhost, - int port, - int timeout, - boolean ssl, - httpRemoteProxyConfig remoteProxyConfig - ) throws IOException { - return getInstance(server,vhost,port,timeout,ssl,remoteProxyConfig,null,null); - } - - /** - * Put back a used instance into the instance pool of httpc. - * - * @param theHttpc The instance of httpc which should be returned to the pool - */ - public static void returnInstance(httpc theHttpc) { - try { - theHttpc.reset(); - httpc.theHttpcPool.returnObject(theHttpc); - } catch (Exception e) { - // we could ignore this error - } - } - /** * Sets wether the content is allowed to be unzipped while getting? * FIXME: The name of this method seems misleading, if I read the usage of @@ -377,15 +254,13 @@ public final class httpc { } /** - * Initialize the httpc-instance with the given data. This method is used, - * if you have to use a proxy to access the pages. This just calls init - * without proxy information and adds the proxy information. + * Initialize the httpc-instance with the given data. * * @param remoteProxyHost * @param remoteProxyPort * @throws IOException */ - private void init( + public httpc( String server, String vhost, int port, @@ -506,9 +381,6 @@ public final class httpc { this.socket.getInputStream()); this.clientOutput = this.socket.getOutputStream(); - - - // if we reached this point, we should have a connection } catch (UnknownHostException e) { if (this.socket != null) { @@ -528,69 +400,7 @@ public final class httpc { return (this.clientOutputByteCount == null)?0:this.clientOutputByteCount.getCount(); } - /** - * This method resets an httpc-instance, so that it can be used for the next - * connection. This is called before the instance is put back to the pool. - * All streams and sockets are closed and set to null. - */ - void reset() { - if (this.clientInput != null) { - try {this.clientInput.close();} catch (Exception e) {} - this.clientInput = null; - } - if (this.clientOutput != null) { - try {this.clientOutput.close();} catch (Exception e) {} - this.clientOutput = null; - } - if (this.socket != null) { - try {this.socket.close();} catch (Exception e) {} - httpc.unregisterOpenSocket(this.socket,this.socketOwner); - this.socket = null; - this.socketOwner = null; - } - - if (this.clientInputByteCount != null) { - this.clientInputByteCount.finish(); - this.clientInputByteCount = null; - } - if (this.clientOutputByteCount != null) { - this.clientOutputByteCount.finish(); - this.clientOutputByteCount = null; - } - - this.adressed_host = null; - this.target_virtual_host = null; - //this.timeout = 0; - - this.remoteProxyUse = false; - this.remoteProxyConfig = null; - this.requestPath = null; - - this.allowContentEncoding = true; - - // shrink readlinebuffer if it is to large - this.readLineBuffer.reset(80); - } - - /** - * Just calls reset to close all connections. - */ public void close() { - reset(); - } - - /** - * If this instance is garbage-collected we check if the object was returned - * to the pool. if not, we invalidate the object from the pool. - * - * @see httpcPool#invalidateObject - */ - protected void finalize() throws Throwable { - if (!(this.removedFromPool)) { - System.err.println("Httpc object was not returned to object pool."); - httpc.theHttpcPool.invalidateObject(this); - } - this.reset(); } /** @@ -939,29 +749,22 @@ public final class httpc { } httpc con = null; - try { - con = httpc.getInstance(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig); - - httpc.response res = con.GET(path, requestHeader); - if (res.status.startsWith("2")) { - if (download == null) { - // stream to byte[] - serverByteBuffer sbb = new serverByteBuffer(); - res.writeContent(sbb, null); - return sbb.getBytes(); - } else { - // stream to file and return null - res.writeContent(null, download); - return null; - } + con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null); + + httpc.response res = con.GET(path, requestHeader); + if (res.status.startsWith("2")) { + if (download == null) { + // stream to byte[] + serverByteBuffer sbb = new serverByteBuffer(); + res.writeContent(sbb, null); + return sbb.getBytes(); + } else { + // stream to file and return null + res.writeContent(null, download); + return null; } - return res.status.getBytes(); - } catch (Exception e) { - throw new IOException(e.getMessage()); - } finally { - if (con != null) httpc.returnInstance(con); } - + return res.status.getBytes(); } public static byte[] singleGET( @@ -1002,24 +805,16 @@ public final class httpc { requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password)); } - httpc con = null; - try { - con = httpc.getInstance(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig); - httpc.response res = con.POST(path, requestHeader, props, files); - - //System.out.println("response=" + res.toString()); - if (res.status.startsWith("2")) { - serverByteBuffer sbb = new serverByteBuffer(); - res.writeContent(sbb, null); - return sbb.getBytes(); - } - return res.status.getBytes(); - } catch (Exception e) { - throw new IOException(e.getMessage()); - } finally { - if (con != null) httpc.returnInstance(con); - } + httpc con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null); + httpc.response res = con.POST(path, requestHeader, props, files); + //System.out.println("response=" + res.toString()); + if (!(res.status.startsWith("2"))) return res.status.getBytes(); + + // read connection body and return body + serverByteBuffer sbb = new serverByteBuffer(); + res.writeContent(sbb, null); + return sbb.getBytes(); } public static byte[] singlePOST( @@ -1161,20 +956,14 @@ public final class httpc { // start connection httpc con = null; - try { - con = httpc.getInstance(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig); - httpc.response res = con.HEAD(path, requestHeader); - if (res.status.startsWith("2")) { - // success - return res.responseHeader; - } - // fail + con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null); + httpc.response res = con.HEAD(path, requestHeader); + if (res.status.startsWith("2")) { + // success return res.responseHeader; - } catch (Exception e) { - throw new IOException(e.getMessage()); - } finally { - if (con != null) httpc.returnInstance(con); } + // fail + return res.responseHeader; } public static byte[] wput( @@ -1456,7 +1245,7 @@ public final class httpc { } // reads in the http header, right now, right here - byte[] b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false); + byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false); if (b == null) { // the server has meanwhile disconnected this.statusCode = 503; @@ -1475,7 +1264,7 @@ public final class httpc { if ((this.statusCode==500)&&(this.statusText.equals("status line parse error"))) { // flush in anything that comes without parsing - while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false); + while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false); return; // in bad mood } @@ -1483,13 +1272,13 @@ public final class httpc { if (this.statusCode == 400) { // bad request // flush in anything that comes without parsing - while ((b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false)).length != 0) {} + while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)).length != 0) {} return; // in bad mood } // at this point we should have a valid response. read in the header properties String key = ""; - while ((b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false)) != null) { + while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)) != null) { if (b.length == 0) break; buffer = new String(b); buffer=buffer.trim(); @@ -1786,86 +1575,3 @@ public class SSLSocketClientWithClientAuth { } } */ - -final class httpcFactory implements org.apache.commons.pool.PoolableObjectFactory { - - public httpcFactory() { - super(); - } - - /** - * @see org.apache.commons.pool.PoolableObjectFactory#makeObject() - */ - public Object makeObject() throws Exception { - return new httpc(); - } - - /** - * @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object) - */ - public void destroyObject(Object obj) { - assert(obj instanceof httpc): "Invalid object type added to pool."; - if (obj instanceof httpc) { - httpc theHttpc = (httpc) obj; - - theHttpc.removedFromPool = true; - } - } - - /** - * @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object) - */ - public boolean validateObject(Object obj) { - assert(obj instanceof httpc): "Invalid object type in pool."; - return true; - } - - /** - * @param obj - * - */ - public void activateObject(Object obj) { - //log.debug(" activateObject..."); - } - - /** - * @param obj - * - */ - public void passivateObject(Object obj) { - assert(obj instanceof httpc): "Invalid object type returned to pool."; - } -} - -final class httpcPool extends GenericObjectPool { - /** - * First constructor. - * @param objFactory - */ - public httpcPool(httpcFactory objFactory) { - super(objFactory); - this.setMaxIdle(75); // Maximum idle threads. - this.setMaxActive(150); // Maximum active threads. - this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs. - //this.setMaxWait(1000); // Wait 1 second till a thread is available - } - - public httpcPool(httpcFactory objFactory, - GenericObjectPool.Config config) { - super(objFactory, config); - } - - /** - * @see org.apache.commons.pool.impl.GenericObjectPool#borrowObject() - */ - public Object borrowObject() throws Exception { - return super.borrowObject(); - } - - /** - * @see org.apache.commons.pool.impl.GenericObjectPool#returnObject(java.lang.Object) - */ - public void returnObject(Object obj) throws Exception { - super.returnObject(obj); - } -} diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 5faedd672..50711a27b 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -719,9 +719,7 @@ public final class httpdProxyHandler { // deleting cached content if (cacheFile.exists()) cacheFile.delete(); handleProxyException(e,remote,conProp,respond,url); - } finally { - if (remote != null) httpc.returnInstance(remote); - } + } } @@ -958,8 +956,6 @@ public final class httpdProxyHandler { httpd.sendRespondHeader(conProp,respond,httpVer,res.statusCode,res.statusText,res.responseHeader); } catch (Exception e) { handleProxyException(e,remote,conProp,respond,url); - } finally { - if (remote != null) httpc.returnInstance(remote); } respond.flush(); @@ -1085,8 +1081,6 @@ public final class httpdProxyHandler { } catch (Exception e) { handleProxyException(e,remote,conProp,respond,url); } finally { - if (remote != null) httpc.returnInstance(remote); - respond.flush(); if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish(); @@ -1134,13 +1128,14 @@ public final class httpdProxyHandler { ) { httpc remoteProxy = null; try { - remoteProxy = httpc.getInstance( + remoteProxy = new httpc( host, host, port, timeout, false, - switchboard.remoteProxyConfig + switchboard.remoteProxyConfig, + null, null ); httpc.response response = remoteProxy.CONNECT(host, port, requestHeader); @@ -1159,8 +1154,6 @@ public final class httpdProxyHandler { } } catch (Exception e) { throw new IOException(e.getMessage()); - } finally { - if (remoteProxy != null) httpc.returnInstance(remoteProxy); } } @@ -1271,13 +1264,14 @@ public final class httpdProxyHandler { } // branch to server/proxy - return httpc.getInstance( + return new httpc( server, server, port, timeout, false, - remProxyConfig + remProxyConfig, + null, null ); } diff --git a/source/de/anomic/plasma/crawler/http/CrawlWorker.java b/source/de/anomic/plasma/crawler/http/CrawlWorker.java index 329d06da1..3fb4a9b0d 100644 --- a/source/de/anomic/plasma/crawler/http/CrawlWorker.java +++ b/source/de/anomic/plasma/crawler/http/CrawlWorker.java @@ -193,7 +193,7 @@ public final class CrawlWorker extends AbstractCrawlWorker { requestHeader.put(httpHeader.ACCEPT_ENCODING, this.acceptEncoding); // open the connection - remote = httpc.getInstance(host, host, port, this.socketTimeout, ssl, this.remoteProxyConfig,"CRAWLER",null); + remote = new httpc(host, host, port, this.socketTimeout, ssl, this.remoteProxyConfig, "CRAWLER", null); // specifying if content encoding is allowed remote.setAllowContentEncoding((this.acceptEncoding != null && this.acceptEncoding.length() > 0)); @@ -315,10 +315,6 @@ public final class CrawlWorker extends AbstractCrawlWorker { // normalizing URL yacyURL redirectionUrl = yacyURL.newURL(this.url, redirectionUrlString); - // returning the used httpc - httpc.returnInstance(remote); - remote = null; - // restart crawling with new url this.log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + this.url.toString()); this.log.logInfo("CRAWLER ..Redirecting request to: " + redirectionUrl); @@ -452,10 +448,6 @@ public final class CrawlWorker extends AbstractCrawlWorker { return null; } - // returning the used httpc - if (remote != null) httpc.returnInstance(remote); - remote = null; - // setting the retry counter to 1 if (crawlingRetryCount > 2) crawlingRetryCount = 2; @@ -467,8 +459,6 @@ public final class CrawlWorker extends AbstractCrawlWorker { addURLtoErrorDB(failreason); } return null; - } finally { - if (remote != null) httpc.returnInstance(remote); } } diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 055f6fe4c..7620cc745 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -398,7 +398,7 @@ public final class plasmaCrawlLURL { yacyURL newUrl = new yacyURL(newUrlStr, null); // doing a http head request to test if the url is correct - theHttpc = httpc.getInstance(newUrl.getHost(), newUrl.getHost(), newUrl.getPort(), 30000, false, plasmaSwitchboard.getSwitchboard().remoteProxyConfig); + theHttpc = new httpc(newUrl.getHost(), newUrl.getHost(), newUrl.getPort(), 30000, false, plasmaSwitchboard.getSwitchboard().remoteProxyConfig, null, null); response res = theHttpc.HEAD(newUrl.getPath(), null); if (res.statusCode == 200) { @@ -416,7 +416,6 @@ public final class plasmaCrawlLURL { } finally { if (theHttpc != null) try { theHttpc.close(); - httpc.returnInstance(theHttpc); } catch (Exception e) { } } } diff --git a/source/de/anomic/plasma/plasmaParser.java b/source/de/anomic/plasma/plasmaParser.java index 2d7e165e1..502cbe879 100644 --- a/source/de/anomic/plasma/plasmaParser.java +++ b/source/de/anomic/plasma/plasmaParser.java @@ -914,13 +914,14 @@ public final class plasmaParser { contentURL = new yacyURL(args[1], null); // downloading the document content - remote = httpc.getInstance( + remote = new httpc( contentURL.getHost(), contentURL.getHost(), contentURL.getPort(), 5000, contentURL.getProtocol().equalsIgnoreCase("https"), - null); + null, + null, null); httpc.response res = remote.GET(contentURL.getFile(), null); if (res.statusCode != 200) { @@ -987,8 +988,6 @@ public final class plasmaParser { } } catch (Exception e) { e.printStackTrace(); - } finally { - if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {} } } diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 37084a213..effdf4886 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -117,7 +117,6 @@ public final class serverCore extends serverAbstractThread implements serverThre private ServerSocket socket; // listener serverLog log; // log object private int timeout; // connection time-out of the socket - private int thresholdSleep = 30000; // after that time a thread is considered as beeing sleeping (30 seconds) serverHandler handlerPrototype; // the command class (a serverHandler) @@ -762,9 +761,6 @@ public final class serverCore extends serverAbstractThread implements serverThre public OutputStream out; // on control output stream, autoflush public int socketTimeout; - private final serverByteBuffer readLineBuffer = new serverByteBuffer(256); - - public Session(ThreadGroup theThreadGroup) { super(theThreadGroup,"Session_created"); } @@ -864,7 +860,7 @@ public final class serverCore extends serverAbstractThread implements serverThre } public byte[] readLine() { - return receive(this.in, this.readLineBuffer, serverCore.this.commandMaxLength, false); + return receive(this.in, serverCore.this.commandMaxLength, false); } /** @@ -890,7 +886,6 @@ public final class serverCore extends serverAbstractThread implements serverThre public void reset() { this.done = true; this.syncObject = null; - this.readLineBuffer.reset(); if (this.commandObj !=null) this.commandObj.reset(); this.userAddress = null; this.userPort = 0; @@ -1174,10 +1169,10 @@ public final class serverCore extends serverAbstractThread implements serverThre } - public static byte[] receive(PushbackInputStream pbis, serverByteBuffer readLineBuffer, int maxSize, boolean logerr) { + public static byte[] receive(PushbackInputStream pbis, int maxSize, boolean logerr) { // reuse an existing linebuffer - readLineBuffer.reset(); + serverByteBuffer readLineBuffer = new serverByteBuffer(80); int bufferSize = 0, b = 0; try { diff --git a/source/de/anomic/server/serverFileUtils.java b/source/de/anomic/server/serverFileUtils.java index 6e3791cc2..4b2e7b511 100644 --- a/source/de/anomic/server/serverFileUtils.java +++ b/source/de/anomic/server/serverFileUtils.java @@ -113,14 +113,18 @@ public final class serverFileUtils { public static void writeX(InputStream source, OutputStream procOS, OutputStream bufferOS) { byte[] buffer = new byte[2048]; - int l; - + int l, c = 0; + while (true) try { l = source.read(buffer, 0, buffer.length); if (l <= 0) break; + c += l; if (procOS != null) procOS.write(buffer, 0, l); if (bufferOS != null) bufferOS.write(buffer, 0, l); - } catch (IOException e) {break;} + } catch (IOException e) { + System.out.println("** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes."); + break; + } // flush the streams if (procOS != null) try { procOS.flush(); } catch (IOException e) {} @@ -130,14 +134,18 @@ public final class serverFileUtils { public static void writeX(Reader source, Writer procOS, Writer bufferOS) { char[] buffer = new char[2048]; - int l; + int l, c= 0; while (true) try{ l = source.read(buffer, 0, buffer.length); if (l <= 0) break; + c += l; if (procOS != null) procOS.write(buffer, 0, l); if (bufferOS != null) bufferOS.write(buffer, 0, l); - } catch (IOException e) {break;} + } catch (IOException e) { + System.out.println("** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes."); + break; + } // flush the streams if (procOS != null) try { procOS.flush(); } catch (IOException e) {} diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 1341d41b7..bfd7a8f88 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -769,13 +769,14 @@ public final class yacySeedDB { httpc remote = null; try { // init httpc - remote = httpc.getInstance( + remote = new httpc( seedURL.getHost(), seedURL.getHost(), seedURL.getPort(), 10000, seedURL.getProtocol().equalsIgnoreCase("https"), - sb.remoteProxyConfig); + sb.remoteProxyConfig, + null, null); // Configure http headers httpHeader reqHeader = new httpHeader(); @@ -801,9 +802,7 @@ public final class yacySeedDB { return nxTools.strings(content,"UTF-8"); } catch (Exception e) { throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage()); - } finally { - if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {} - } + } } private String checkCache(ArrayList uv, ArrayList check) { diff --git a/source/yacy.java b/source/yacy.java index d4df71164..67dcb4e5c 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -509,7 +509,7 @@ public final class yacy { httpHeader requestHeader = new httpHeader(); requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify try { - httpc con = httpc.getInstance("localhost", "localhost", port, 10000, false, null); + httpc con = new httpc("localhost", "localhost", port, 10000, false, null, null, null); httpc.response res = con.GET("Steering.html?shutdown=", requestHeader); // read response