boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

cleanup of unmaintained and outdated performance methods:

Browse Source 
removed object pools in httpc. Object pooling is not recommended,
if the creation of the object is not time-intensive. Object pools are only useful,
if there is much computation necessary to create some basic data that is stored
in the object pool and can be re-used. This does not apply to object pools in YaCy.
Object pooling of client sessions would make sense if they would allow re-use of
living connections to other yacy clients. But every connection is closed after usage
of an object in the client pool, therefore the YaCy server client objects are not such
that hold hardware/network-allocated entities.
See:
http://www.javaperformancetuning.com/news/qotm033.shtml
http://java.sun.com/docs/hotspot/HotSpotFAQ.html#gc_pooling
http://docs.sun.com/source/816-7159-10/pt_chap5.html
http://www.microjava.com/articles/techtalk/recylcle2


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4106 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 18 years ago
parent 3cb9cdc9be
commit 1488769e1f
12 changed files with 109 additions and 428 deletions
Show Stats Download Patch File Download Diff File

5
htroot/CrawlURLFetch_p.java
Unescape View File

@ -507,13 +507,13 @@ public class CrawlURLFetch_p {
if (url == null) return null;
String[] r = null;
try {
httpc con = httpc.getInstance(
httpc con = new httpc(
url.getHost(),
url.getHost(),
url.getPort(),
15000,
url.getProtocol().equals("https"),
plasmaSwitchboard.getSwitchboard().remoteProxyConfig);
plasmaSwitchboard.getSwitchboard().remoteProxyConfig, null, null);
httpHeader header = new httpHeader();
header.put(httpHeader.ACCEPT_ENCODING, "US-ASCII");
@ -531,7 +531,6 @@ public class CrawlURLFetch_p {
if (encoding == null) encoding = "US-ASCII";
r = parseText(new String(sbb.getBytes(), encoding));
}
httpc.returnInstance(con);
} catch (IOException e) { }
return r;
}

6
source/de/anomic/data/SitemapParser.java
Unescape View File

@ -174,13 +174,13 @@ public class SitemapParser extends DefaultHandler {
// download document
httpc remote = null;
try {
remote = httpc.getInstance(
remote = new httpc(
this.siteMapURL.getHost(),
this.siteMapURL.getHost(),
this.siteMapURL.getPort(),
5000,
this.siteMapURL.getProtocol().equalsIgnoreCase("https"),
switchboard.remoteProxyConfig);
switchboard.remoteProxyConfig, null, null);
httpc.response res = remote.GET(this.siteMapURL.getFile(), null);
if (res.statusCode != 200) {
@ -211,8 +211,6 @@ public class SitemapParser extends DefaultHandler {
saxParser.parse(this.counterStream, this);
} catch (Exception e) {
this.logger.logWarning("Unable to parse sitemap file " + this.siteMapURL,e);
} finally {
if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {/* ignore this */}
}
}

12
source/de/anomic/data/robotsParser.java
Unescape View File

@ -386,7 +386,7 @@ public final class robotsParser{
downloadStart = System.currentTimeMillis();
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
//TODO: adding Traffic statistic for robots download?
con = httpc.getInstance(robotsURL.getHost(), robotsURL.getHost(), robotsURL.getPort(), 10000, robotsURL.getProtocol().equalsIgnoreCase("https"), sb.remoteProxyConfig);
con = new httpc(robotsURL.getHost(), robotsURL.getHost(), robotsURL.getPort(), 10000, robotsURL.getProtocol().equalsIgnoreCase("https"), sb.remoteProxyConfig, null, null);
// if we previously have downloaded this robots.txt then we can set the if-modified-since header
httpHeader reqHeaders = new httpHeader();
@ -447,11 +447,7 @@ public final class robotsParser{
redirectionUrlString = redirectionUrlString.trim();
// generating the new URL object
yacyURL redirectionUrl = yacyURL.newURL(robotsURL, redirectionUrlString);
// returning the used httpc
httpc.returnInstance(con);
con = null;
yacyURL redirectionUrl = yacyURL.newURL(robotsURL, redirectionUrlString);
// following the redirection
serverLog.logFinest("ROBOTS","Redirection detected for robots.txt with URL '" + robotsURL + "'." +
@ -467,9 +463,7 @@ public final class robotsParser{
}
} catch (Exception e) {
throw e;
} finally {
if (con != null) httpc.returnInstance(con);
}
}
return new Object[]{new Boolean(accessCompletelyRestricted),robotsTxt,eTag,lastMod};
}
}

432
source/de/anomic/http/httpc.java
Unescape View File

@ -74,8 +74,6 @@ import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverCore;
@ -91,8 +89,8 @@ import de.anomic.yacy.yacyURL;
* libraries, it is still necessary to implement the network interface since
* otherwise there is no access to the HTTP/1.0 / HTTP/1.1 header information
* that comes along each connection.
* FIXME: Add some information about the usage of the threadpool.
*/
public final class httpc {
// some constants
@ -103,87 +101,30 @@ public final class httpc {
*/
public static final String GZIP_POST_BODY = "GZIP_POST_BODY";
// statics
// final statics
private static final String vDATE = "20040602";
public static String userAgent;
private static final int terminalMaxLength = 30000;
private static final TimeZone GMTTimeZone = TimeZone.getTimeZone("GMT");
/**
* This string is initialized on loading of this class and contains
* information about the current OS.
*/
public static String systemOST;
// --- The GMT standard date format used in the HTTP protocol
private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
static {
HTTPGMTFormatter.setTimeZone(GMTTimeZone);
}
/**
* A Object Pool containing all pooled httpc-objects.
* @see httpcPool
*/
private static final httpcPool theHttpcPool;
// class variables
private Socket socket = null; // client socket for commands
private Thread socketOwner = null;
private String adressed_host = null;
private int adressed_port = 80;
private String target_virtual_host = null;
// output and input streams for client control connection
PushbackInputStream clientInput = null;
private OutputStream clientOutput = null;
private httpdByteCountInputStream clientInputByteCount = null;
private httpdByteCountOutputStream clientOutputByteCount = null;
private boolean remoteProxyUse = false;
private httpRemoteProxyConfig remoteProxyConfig = null;
private static final HashMap reverseMappingCache = new HashMap();
private static final HashMap openSocketLookupTable = new HashMap();
String requestPath = null;
private boolean allowContentEncoding = true;
public static boolean yacyDebugMode = false;
static final HashMap reverseMappingCache = new HashMap();
// defined during set-up of switchboard
public static boolean yacyDebugMode = false;
/**
* Indicates if the current object was removed from pool because the maximum limit
* was exceeded.
*/
boolean removedFromPool = false;
static SSLSocketFactory theSSLSockFactory = null;
// statics to be defined in static section below
private static SSLSocketFactory theSSLSockFactory = null;
public static String systemOST;
public static String userAgent;
static {
// set the time zone
HTTPGMTFormatter.setTimeZone(GMTTimeZone); // The GMT standard date format used in the HTTP protocol
// set time-out of InetAddress.getByName cache ttl
java.security.Security.setProperty("networkaddress.cache.ttl" , "60");
java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0");
// Configuring the httpc object pool
// implementation of session thread pool
GenericObjectPool.Config config = new GenericObjectPool.Config();
// The maximum number of active connections that can be allocated from pool at the same time,
// 0 for no limit
config.maxActive = 150;
// The maximum number of idle connections connections in the pool
// 0 = no limit.
config.maxIdle = 75;
config.minIdle = 10;
config.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
config.minEvictableIdleTimeMillis = 30000;
theHttpcPool = new httpcPool(new httpcFactory(),config);
// initializing a dummy trustManager to enable https connections
// Create a trust manager that does not validate certificate chains
@ -218,8 +159,7 @@ public final class httpc {
HttpsURLConnection.setDefaultHostnameVerifier(hv);
} catch (Exception e) {
}
// provide system information for client identification
String loc = System.getProperty("user.timezone", "nowhere");
int p = loc.indexOf("/");
@ -233,13 +173,27 @@ public final class httpc {
userAgent = "yacy (www.yacy.net; v" + vDATE + "; " + systemOST + ")";
}
/**
* A reusable readline buffer
* @see serverByteBuffer
*/
final serverByteBuffer readLineBuffer = new serverByteBuffer(100);
// class variables
private Socket socket = null; // client socket for commands
private Thread socketOwner = null;
private String adressed_host = null;
private int adressed_port = 80;
private String target_virtual_host = null;
// output and input streams for client control connection
private PushbackInputStream clientInput = null;
private OutputStream clientOutput = null;
private httpdByteCountInputStream clientInputByteCount = null;
private httpdByteCountOutputStream clientOutputByteCount = null;
private static final HashMap openSocketLookupTable = new HashMap();
private boolean remoteProxyUse = false;
private httpRemoteProxyConfig remoteProxyConfig = null;
private String requestPath = null;
private boolean allowContentEncoding = true;
/**
* Convert the status of this class into an String object to output it.
@ -249,83 +203,6 @@ public final class httpc {
((this.remoteProxyUse) ? " via " + adressed_host : "");
}
/**
* This method gets a new httpc instance from the object pool and
* initializes it with the given parameters. Use this method if you have to
* use a proxy to access the pages.
*
* @param server
* @param port
* @param timeout
* @param ssl
* @param remoteProxyHost
* @param remoteProxyPort
* @throws IOException
* @see httpc#init
*/
public static httpc getInstance(
String server,
String vhost,
int port,
int timeout,
boolean ssl,
httpRemoteProxyConfig remoteProxyConfig,
String incomingByteCountAccounting,
String outgoingByteCountAccounting
) throws IOException {
httpc newHttpc;
try {
// fetching a new httpc from the object pool
newHttpc = (httpc) httpc.theHttpcPool.borrowObject();
} catch (Exception e) {
throw new IOException("Unable to initialize a new httpc. " + e.getMessage());
}
// initialize it
try {
newHttpc.init(
server,
vhost,
port,
timeout,
ssl,
remoteProxyConfig,
incomingByteCountAccounting,
outgoingByteCountAccounting
);
} catch (IOException e) {
try{ httpc.theHttpcPool.returnObject(newHttpc); } catch (Exception e1) {}
throw e;
}
return newHttpc;
}
public static httpc getInstance(
String server,
String vhost,
int port,
int timeout,
boolean ssl,
httpRemoteProxyConfig remoteProxyConfig
) throws IOException {
return getInstance(server,vhost,port,timeout,ssl,remoteProxyConfig,null,null);
}
/**
* Put back a used instance into the instance pool of httpc.
*
* @param theHttpc The instance of httpc which should be returned to the pool
*/
public static void returnInstance(httpc theHttpc) {
try {
theHttpc.reset();
httpc.theHttpcPool.returnObject(theHttpc);
} catch (Exception e) {
// we could ignore this error
}
}
/**
* Sets wether the content is allowed to be unzipped while getting?
* FIXME: The name of this method seems misleading, if I read the usage of
@ -377,15 +254,13 @@ public final class httpc {
}
/**
* Initialize the httpc-instance with the given data. This method is used,
* if you have to use a proxy to access the pages. This just calls init
* without proxy information and adds the proxy information.
* Initialize the httpc-instance with the given data.
*
* @param remoteProxyHost
* @param remoteProxyPort
* @throws IOException
*/
private void init(
public httpc(
String server,
String vhost,
int port,
@ -506,9 +381,6 @@ public final class httpc {
this.socket.getInputStream());
this.clientOutput = this.socket.getOutputStream();
// if we reached this point, we should have a connection
} catch (UnknownHostException e) {
if (this.socket != null) {
@ -528,69 +400,7 @@ public final class httpc {
return (this.clientOutputByteCount == null)?0:this.clientOutputByteCount.getCount();
}
/**
* This method resets an httpc-instance, so that it can be used for the next
* connection. This is called before the instance is put back to the pool.
* All streams and sockets are closed and set to null.
*/
void reset() {
if (this.clientInput != null) {
try {this.clientInput.close();} catch (Exception e) {}
this.clientInput = null;
}
if (this.clientOutput != null) {
try {this.clientOutput.close();} catch (Exception e) {}
this.clientOutput = null;
}
if (this.socket != null) {
try {this.socket.close();} catch (Exception e) {}
httpc.unregisterOpenSocket(this.socket,this.socketOwner);
this.socket = null;
this.socketOwner = null;
}
if (this.clientInputByteCount != null) {
this.clientInputByteCount.finish();
this.clientInputByteCount = null;
}
if (this.clientOutputByteCount != null) {
this.clientOutputByteCount.finish();
this.clientOutputByteCount = null;
}
this.adressed_host = null;
this.target_virtual_host = null;
//this.timeout = 0;
this.remoteProxyUse = false;
this.remoteProxyConfig = null;
this.requestPath = null;
this.allowContentEncoding = true;
// shrink readlinebuffer if it is to large
this.readLineBuffer.reset(80);
}
/**
* Just calls reset to close all connections.
*/
public void close() {
reset();
}
/**
* If this instance is garbage-collected we check if the object was returned
* to the pool. if not, we invalidate the object from the pool.
*
* @see httpcPool#invalidateObject
*/
protected void finalize() throws Throwable {
if (!(this.removedFromPool)) {
System.err.println("Httpc object was not returned to object pool.");
httpc.theHttpcPool.invalidateObject(this);
}
this.reset();
}
/**
@ -939,29 +749,22 @@ public final class httpc {
}
httpc con = null;
try {
con = httpc.getInstance(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig);
httpc.response res = con.GET(path, requestHeader);
if (res.status.startsWith("2")) {
if (download == null) {
// stream to byte[]
serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null);
return sbb.getBytes();
} else {
// stream to file and return null
res.writeContent(null, download);
return null;
}
con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc.response res = con.GET(path, requestHeader);
if (res.status.startsWith("2")) {
if (download == null) {
// stream to byte[]
serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null);
return sbb.getBytes();
} else {
// stream to file and return null
res.writeContent(null, download);
return null;
}
return res.status.getBytes();
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
return res.status.getBytes();
}
public static byte[] singleGET(
@ -1002,24 +805,16 @@ public final class httpc {
requestHeader.put(httpHeader.AUTHORIZATION, kelondroBase64Order.standardCoder.encodeString(user + ":" + password));
}
httpc con = null;
try {
con = httpc.getInstance(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig);
httpc.response res = con.POST(path, requestHeader, props, files);
//System.out.println("response=" + res.toString());
if (res.status.startsWith("2")) {
serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null);
return sbb.getBytes();
}
return res.status.getBytes();
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
httpc con = new httpc(realhost, virtualhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc.response res = con.POST(path, requestHeader, props, files);
//System.out.println("response=" + res.toString());
if (!(res.status.startsWith("2"))) return res.status.getBytes();
// read connection body and return body
serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null);
return sbb.getBytes();
}
public static byte[] singlePOST(
@ -1161,20 +956,14 @@ public final class httpc {
// start connection
httpc con = null;
try {
con = httpc.getInstance(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig);
httpc.response res = con.HEAD(path, requestHeader);
if (res.status.startsWith("2")) {
// success
return res.responseHeader;
}
// fail
con = new httpc(realhost, vhost, port, timeout, ssl, theRemoteProxyConfig, null, null);
httpc.response res = con.HEAD(path, requestHeader);
if (res.status.startsWith("2")) {
// success
return res.responseHeader;
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (con != null) httpc.returnInstance(con);
}
// fail
return res.responseHeader;
}
public static byte[] wput(
@ -1456,7 +1245,7 @@ public final class httpc {
}
// reads in the http header, right now, right here
byte[] b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false);
byte[] b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
if (b == null) {
// the server has meanwhile disconnected
this.statusCode = 503;
@ -1475,7 +1264,7 @@ public final class httpc {
if ((this.statusCode==500)&&(this.statusText.equals("status line parse error"))) {
// flush in anything that comes without parsing
while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false);
while ((b != null) && (b.length != 0)) b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
return; // in bad mood
}
@ -1483,13 +1272,13 @@ public final class httpc {
if (this.statusCode == 400) {
// bad request
// flush in anything that comes without parsing
while ((b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false)).length != 0) {}
while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)).length != 0) {}
return; // in bad mood
}
// at this point we should have a valid response. read in the header properties
String key = "";
while ((b = serverCore.receive(httpc.this.clientInput, httpc.this.readLineBuffer, terminalMaxLength, false)) != null) {
while ((b = serverCore.receive(httpc.this.clientInput, terminalMaxLength, false)) != null) {
if (b.length == 0) break;
buffer = new String(b);
buffer=buffer.trim();
@ -1786,86 +1575,3 @@ public class SSLSocketClientWithClientAuth {
}
}
*/
final class httpcFactory implements org.apache.commons.pool.PoolableObjectFactory {
public httpcFactory() {
super();
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#makeObject()
*/
public Object makeObject() throws Exception {
return new httpc();
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)
*/
public void destroyObject(Object obj) {
assert(obj instanceof httpc): "Invalid object type added to pool.";
if (obj instanceof httpc) {
httpc theHttpc = (httpc) obj;
theHttpc.removedFromPool = true;
}
}
/**
* @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object)
*/
public boolean validateObject(Object obj) {
assert(obj instanceof httpc): "Invalid object type in pool.";
return true;
}
/**
* @param obj
*
*/
public void activateObject(Object obj) {
//log.debug(" activateObject...");
}
/**
* @param obj
*
*/
public void passivateObject(Object obj) {
assert(obj instanceof httpc): "Invalid object type returned to pool.";
}
}
final class httpcPool extends GenericObjectPool {
/**
* First constructor.
* @param objFactory
*/
public httpcPool(httpcFactory objFactory) {
super(objFactory);
this.setMaxIdle(75); // Maximum idle threads.
this.setMaxActive(150); // Maximum active threads.
this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs.
//this.setMaxWait(1000); // Wait 1 second till a thread is available
}
public httpcPool(httpcFactory objFactory,
GenericObjectPool.Config config) {
super(objFactory, config);
}
/**
* @see org.apache.commons.pool.impl.GenericObjectPool#borrowObject()
*/
public Object borrowObject() throws Exception {
return super.borrowObject();
}
/**
* @see org.apache.commons.pool.impl.GenericObjectPool#returnObject(java.lang.Object)
*/
public void returnObject(Object obj) throws Exception {
super.returnObject(obj);
}
}

20
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -719,9 +719,7 @@ public final class httpdProxyHandler {
// deleting cached content
if (cacheFile.exists()) cacheFile.delete();
handleProxyException(e,remote,conProp,respond,url);
} finally {
if (remote != null) httpc.returnInstance(remote);
}
}
}
@ -958,8 +956,6 @@ public final class httpdProxyHandler {
httpd.sendRespondHeader(conProp,respond,httpVer,res.statusCode,res.statusText,res.responseHeader);
} catch (Exception e) {
handleProxyException(e,remote,conProp,respond,url);
} finally {
if (remote != null) httpc.returnInstance(remote);
}
respond.flush();
@ -1085,8 +1081,6 @@ public final class httpdProxyHandler {
} catch (Exception e) {
handleProxyException(e,remote,conProp,respond,url);
} finally {
if (remote != null) httpc.returnInstance(remote);
respond.flush();
if (respond instanceof httpdByteCountOutputStream) ((httpdByteCountOutputStream)respond).finish();
@ -1134,13 +1128,14 @@ public final class httpdProxyHandler {
) {
httpc remoteProxy = null;
try {
remoteProxy = httpc.getInstance(
remoteProxy = new httpc(
host,
host,
port,
timeout,
false,
switchboard.remoteProxyConfig
switchboard.remoteProxyConfig,
null, null
);
httpc.response response = remoteProxy.CONNECT(host, port, requestHeader);
@ -1159,8 +1154,6 @@ public final class httpdProxyHandler {
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
if (remoteProxy != null) httpc.returnInstance(remoteProxy);
}
}
@ -1271,13 +1264,14 @@ public final class httpdProxyHandler {
}
// branch to server/proxy
return httpc.getInstance(
return new httpc(
server,
server,
port,
timeout,
false,
remProxyConfig
remProxyConfig,
null, null
);
}

12
source/de/anomic/plasma/crawler/http/CrawlWorker.java
Unescape View File

@ -193,7 +193,7 @@ public final class CrawlWorker extends AbstractCrawlWorker {
requestHeader.put(httpHeader.ACCEPT_ENCODING, this.acceptEncoding);
// open the connection
remote = httpc.getInstance(host, host, port, this.socketTimeout, ssl, this.remoteProxyConfig,"CRAWLER",null);
remote = new httpc(host, host, port, this.socketTimeout, ssl, this.remoteProxyConfig, "CRAWLER", null);
// specifying if content encoding is allowed
remote.setAllowContentEncoding((this.acceptEncoding != null && this.acceptEncoding.length() > 0));
@ -315,10 +315,6 @@ public final class CrawlWorker extends AbstractCrawlWorker {
// normalizing URL
yacyURL redirectionUrl = yacyURL.newURL(this.url, redirectionUrlString);
// returning the used httpc
httpc.returnInstance(remote);
remote = null;
// restart crawling with new url
this.log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + this.url.toString());
this.log.logInfo("CRAWLER ..Redirecting request to: " + redirectionUrl);
@ -452,10 +448,6 @@ public final class CrawlWorker extends AbstractCrawlWorker {
return null;
}
// returning the used httpc
if (remote != null) httpc.returnInstance(remote);
remote = null;
// setting the retry counter to 1
if (crawlingRetryCount > 2) crawlingRetryCount = 2;
@ -467,8 +459,6 @@ public final class CrawlWorker extends AbstractCrawlWorker {
addURLtoErrorDB(failreason);
}
return null;
} finally {
if (remote != null) httpc.returnInstance(remote);
}
}

3
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -398,7 +398,7 @@ public final class plasmaCrawlLURL {
yacyURL newUrl = new yacyURL(newUrlStr, null);
// doing a http head request to test if the url is correct
theHttpc = httpc.getInstance(newUrl.getHost(), newUrl.getHost(), newUrl.getPort(), 30000, false, plasmaSwitchboard.getSwitchboard().remoteProxyConfig);
theHttpc = new httpc(newUrl.getHost(), newUrl.getHost(), newUrl.getPort(), 30000, false, plasmaSwitchboard.getSwitchboard().remoteProxyConfig, null, null);
response res = theHttpc.HEAD(newUrl.getPath(), null);
if (res.statusCode == 200) {
@ -416,7 +416,6 @@ public final class plasmaCrawlLURL {
} finally {
if (theHttpc != null) try {
theHttpc.close();
httpc.returnInstance(theHttpc);
} catch (Exception e) { }
}
}

7
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -914,13 +914,14 @@ public final class plasmaParser {
contentURL = new yacyURL(args[1], null);
// downloading the document content
remote = httpc.getInstance(
remote = new httpc(
contentURL.getHost(),
contentURL.getHost(),
contentURL.getPort(),
5000,
contentURL.getProtocol().equalsIgnoreCase("https"),
null);
null,
null, null);
httpc.response res = remote.GET(contentURL.getFile(), null);
if (res.statusCode != 200) {
@ -987,8 +988,6 @@ public final class plasmaParser {
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {}
}
}

11
source/de/anomic/server/serverCore.java
Unescape View File

@ -117,7 +117,6 @@ public final class serverCore extends serverAbstractThread implements serverThre
private ServerSocket socket; // listener
serverLog log; // log object
private int timeout; // connection time-out of the socket
private int thresholdSleep = 30000; // after that time a thread is considered as beeing sleeping (30 seconds)
serverHandler handlerPrototype; // the command class (a serverHandler)
@ -762,9 +761,6 @@ public final class serverCore extends serverAbstractThread implements serverThre
public OutputStream out; // on control output stream, autoflush
public int socketTimeout;
private final serverByteBuffer readLineBuffer = new serverByteBuffer(256);
public Session(ThreadGroup theThreadGroup) {
super(theThreadGroup,"Session_created");
}
@ -864,7 +860,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
}
public byte[] readLine() {
return receive(this.in, this.readLineBuffer, serverCore.this.commandMaxLength, false);
return receive(this.in, serverCore.this.commandMaxLength, false);
}
/**
@ -890,7 +886,6 @@ public final class serverCore extends serverAbstractThread implements serverThre
public void reset() {
this.done = true;
this.syncObject = null;
this.readLineBuffer.reset();
if (this.commandObj !=null) this.commandObj.reset();
this.userAddress = null;
this.userPort = 0;
@ -1174,10 +1169,10 @@ public final class serverCore extends serverAbstractThread implements serverThre
}
public static byte[] receive(PushbackInputStream pbis, serverByteBuffer readLineBuffer, int maxSize, boolean logerr) {
public static byte[] receive(PushbackInputStream pbis, int maxSize, boolean logerr) {
// reuse an existing linebuffer
readLineBuffer.reset();
serverByteBuffer readLineBuffer = new serverByteBuffer(80);
int bufferSize = 0, b = 0;
try {

18
source/de/anomic/server/serverFileUtils.java
Unescape View File

@ -113,14 +113,18 @@ public final class serverFileUtils {
public static void writeX(InputStream source, OutputStream procOS, OutputStream bufferOS) {
byte[] buffer = new byte[2048];
int l;
int l, c = 0;
while (true) try {
l = source.read(buffer, 0, buffer.length);
if (l <= 0) break;
c += l;
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
} catch (IOException e) {break;}
} catch (IOException e) {
System.out.println("** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes.");
break;
}
// flush the streams
if (procOS != null) try { procOS.flush(); } catch (IOException e) {}
@ -130,14 +134,18 @@ public final class serverFileUtils {
public static void writeX(Reader source, Writer procOS, Writer bufferOS) {
char[] buffer = new char[2048];
int l;
int l, c= 0;
while (true) try{
l = source.read(buffer, 0, buffer.length);
if (l <= 0) break;
c += l;
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
} catch (IOException e) {break;}
} catch (IOException e) {
System.out.println("** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes.");
break;
}
// flush the streams
if (procOS != null) try { procOS.flush(); } catch (IOException e) {}

9
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -769,13 +769,14 @@ public final class yacySeedDB {
httpc remote = null;
try {
// init httpc
remote = httpc.getInstance(
remote = new httpc(
seedURL.getHost(),
seedURL.getHost(),
seedURL.getPort(),
10000,
seedURL.getProtocol().equalsIgnoreCase("https"),
sb.remoteProxyConfig);
sb.remoteProxyConfig,
null, null);
// Configure http headers
httpHeader reqHeader = new httpHeader();
@ -801,9 +802,7 @@ public final class yacySeedDB {
return nxTools.strings(content,"UTF-8");
} catch (Exception e) {
throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
} finally {
if (remote != null) try { httpc.returnInstance(remote); } catch (Exception e) {}
}
}
}
private String checkCache(ArrayList uv, ArrayList check) {

2
source/yacy.java
Unescape View File

@ -509,7 +509,7 @@ public final class yacy {
httpHeader requestHeader = new httpHeader();
requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify
try {
httpc con = httpc.getInstance("localhost", "localhost", port, 10000, false, null);
httpc con = new httpc("localhost", "localhost", port, 10000, false, null, null, null);
httpc.response res = con.GET("Steering.html?shutdown=", requestHeader);
// read response

Write Preview
Loading…
Cancel
Save