boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

cleaned up superfluous classes after sixcoolers migration to HttpComponents-Client-4.x

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7062 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent dcd9065c84
commit d0fb6bc2bc
12 changed files with 1 additions and 1820 deletions
Show Stats Download Patch File Download Diff File

109
source/de/anomic/http/client/AcceptEverythingSSLProtcolSocketFactory.java
Unescape View File

@ -1,109 +0,0 @@
package de.anomic.http.client;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.UnknownHostException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import net.yacy.kelondro.logging.Log;
import org.apache.commons.httpclient.ConnectTimeoutException;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
/**
* accepts every Certificate
*
* @author danielr
* @since 12.05.2008
*/
class AcceptEverythingSSLProtcolSocketFactory implements SecureProtocolSocketFactory {
private SSLContext sslContext = null;
/**
* constructor
*/
public AcceptEverythingSSLProtcolSocketFactory() {
super();
try {
sslContext = SSLContext.getInstance("SSL");
sslContext.init(null, new TrustManager[] { new AcceptEverythingTrustManager() }, null);
} catch (final NoSuchAlgorithmException e) {
// SSL should be supported
Log.logException(e);
} catch (final KeyManagementException e) {
Log.logException(e);
} catch (final KeyStoreException e) {
// should never happen, because we don't use a keystore
Log.logException(e);
}
}
/*
* (non-Javadoc)
*
* @see org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory#createSocket(java.net.Socket,
* java.lang.String, int, boolean)
*/
public Socket createSocket(final Socket socket, final String host, final int port, final boolean autoClose)
throws IOException, UnknownHostException {
return sslContext.getSocketFactory().createSocket(socket, host, port, autoClose);
}
/*
* (non-Javadoc)
*
* @see org.apache.commons.httpclient.protocol.ProtocolSocketFactory#createSocket(java.lang.String, int)
*/
public Socket createSocket(final String host, final int port) throws IOException, UnknownHostException {
return sslContext.getSocketFactory().createSocket(host, port);
}
/*
* (non-Javadoc)
*
* @see org.apache.commons.httpclient.protocol.ProtocolSocketFactory#createSocket(java.lang.String, int,
* java.net.InetAddress, int)
*/
public Socket createSocket(final String host, final int port, final InetAddress localAddress, final int localPort)
throws IOException, UnknownHostException {
return sslContext.getSocketFactory().createSocket(host, port, localAddress, localPort);
}
/*
* (non-Javadoc)
*
* @see org.apache.commons.httpclient.protocol.ProtocolSocketFactory#createSocket(java.lang.String, int,
* java.net.InetAddress, int, org.apache.commons.httpclient.params.HttpConnectionParams)
*/
public Socket createSocket(final String host, final int port, final InetAddress localAddress, final int localPort,
final HttpConnectionParams params) throws IOException, UnknownHostException, ConnectTimeoutException {
if (params == null) {
throw new IllegalArgumentException("Parameters may not be null");
}
final Socket socket = sslContext.getSocketFactory().createSocket();
// apply params
if (params.getLinger() > -1) {
socket.setSoLinger((params.getLinger() != 0), params.getLinger());
}
if (params.getReceiveBufferSize() > 0) {
socket.setReceiveBufferSize(params.getReceiveBufferSize());
}
if (params.getSendBufferSize() > 0) {
socket.setSendBufferSize(params.getSendBufferSize());
}
socket.setSoTimeout(params.getSoTimeout());
socket.setTcpNoDelay(params.getTcpNoDelay());
socket.bind(new InetSocketAddress(localAddress, localPort));
socket.connect(new InetSocketAddress(host, port), params.getConnectionTimeout());
return socket;
}
}

47
source/de/anomic/http/client/AcceptEverythingTrustManager.java
Unescape View File

@ -1,47 +0,0 @@
package de.anomic.http.client;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import javax.net.ssl.X509TrustManager;
import net.yacy.kelondro.logging.Log;
/**
* trust every server
*
* @author daniel
*
*/
class AcceptEverythingTrustManager extends EasyX509TrustManager implements X509TrustManager {
/**
* constructor
*
* @param keystore
* @throws NoSuchAlgorithmException
* @throws KeyStoreException
*/
public AcceptEverythingTrustManager() throws NoSuchAlgorithmException, KeyStoreException {
super(null);
}
/*
* (non-Javadoc)
*
* @see javax.net.ssl.X509TrustManager#checkServerTrusted(java.security.cert.X509Certificate[], java.lang.String)
*/
@Override
public void checkServerTrusted(final X509Certificate[] chain, final String authType) throws CertificateException {
try {
super.checkServerTrusted(chain, authType);
} catch (final Exception e) {
// trusted but logged
Log.logWarning("HTTPC", "trusting SSL certificate with " + e.getClass() + ": " + e.getMessage());
}
}
}

742
source/de/anomic/http/client/Client.java
Unescape View File

@ -1,742 +0,0 @@
// JakartaCommonsHttpClient.java
// (C) 2008 by Daniel Raap; danielr@users.berlios.de
// first published 2.4.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http.client;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.ProxySettings;
import net.yacy.kelondro.order.Base64Order;
import org.apache.commons.httpclient.ConnectMethod;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource;
import org.apache.commons.httpclient.methods.multipart.FilePart;
import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
import org.apache.commons.httpclient.methods.multipart.Part;
import org.apache.commons.httpclient.params.DefaultHttpParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
import de.anomic.crawler.Latency;
import de.anomic.http.server.HeaderFramework;
import de.anomic.http.server.RequestHeader;
import de.anomic.http.server.ResponseContainer;
import de.anomic.http.server.ResponseHeader;
/**
* HttpClient implementation which uses Jakarta Commons HttpClient 3.x {@link http://hc.apache.org/httpclient-3.x/}
*
* @author danielr
* @deprecated please use net.yacy.cora.protocol.Client instead
*
*/
public class Client {
/**
* "the HttpClient instance and connection manager should be shared among all threads for maximum efficiency."
* (Concurrent execution of HTTP methods, http://hc.apache.org/httpclient-3.x/performance.html)
*/
private static MultiThreadedHttpConnectionManager conManager = null;
private static HttpClient apacheHttpClient = null;
// last ; must be before location (this is parsed)
private final static String jakartaUserAgent = " " +
((String) DefaultHttpParams.getDefaultParams().getParameter(HttpMethodParams.USER_AGENT)).replace(';', ':');
static {
/**
* set options for client
*/
MultiThreadedHttpConnectionManager.shutdownAll();
initConnectionManager();
// accept self-signed or untrusted certificates
Protocol.registerProtocol("https", new Protocol("https",
(ProtocolSocketFactory) new AcceptEverythingSSLProtcolSocketFactory(), 443));
/**
* set network timeout properties. see: http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html These
* properties specify the default connect and read timeout (resp.) for the protocol handler used by
* java.net.URLConnection. the java.net.URLConnection is also used by JakartaCommons HttpClient, see
* http://hc.apache.org/httpclient-3.x/apidocs/org/apache/commons/httpclient/util/HttpURLConnection.html
*/
// specify the timeout, in milliseconds, to establish the connection to the host.
// For HTTP connections, it is the timeout when establishing the connection to the HTTP server.
System.setProperty("sun.net.client.defaultConnectTimeout", "10000");
// specify the response timeout, in milliseconds, when reading from an input stream
// after a connection is established with a resource
System.setProperty("sun.net.client.defaultReadTimeout", "60000");
}
public static void initConnectionManager() {
if (conManager != null) {
conManager.closeIdleConnections(0);
conManager.deleteClosedConnections();
conManager.shutdown();
}
conManager = new MultiThreadedHttpConnectionManager();
/**
* set options for connection manager
*/
// conManager.getParams().setDefaultMaxConnectionsPerHost(4); // default 2
HostConfiguration localHostConfiguration = new HostConfiguration();
conManager.getParams().setMaxTotalConnections(50); // Proxy may need many connections
conManager.getParams().setConnectionTimeout(60000); // set a default timeout
conManager.getParams().setDefaultMaxConnectionsPerHost(2);
localHostConfiguration.setHost("0:0:0:0:0:0:0:1%0");
conManager.getParams().setMaxConnectionsPerHost(localHostConfiguration, 100);
localHostConfiguration.setHost("localhost");
conManager.getParams().setMaxConnectionsPerHost(localHostConfiguration, 100);
localHostConfiguration.setHost("127.0.0.1");
conManager.getParams().setMaxConnectionsPerHost(localHostConfiguration, 100);
conManager.getParams().setReceiveBufferSize(16 * 1024 * 1024); // set this high to avoid storage in temporary files
conManager.getParams().setStaleCheckingEnabled(true);
conManager.getParams().setSoTimeout(30000);
conManager.getParams().setTcpNoDelay(true); // there is enough bandwith these days
apacheHttpClient = new HttpClient(conManager);
// only one retry
apacheHttpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(1, false));
// simple user agent
setUserAgent("yacy (www.yacy.net; " + getSystemOST() + ")");
}
/**
* every x milliseconds do a cleanup (close old connections)
*
* minimal intervall the cleanUp is done (in this time after a cleanup no second one is done)
*
* this is the time the method is callable, not the time it is called
*/
private static final int cleanupIntervall = 15000;
/**
* close connections when they are not used for this time
*
* or otherwise: hold connections this time open to reuse them
*/
private static final long closeConnectionsAfterMillis = 12000;
/**
* time the last cleanup was started
*/
private static long lastCleanup = 0;
private Header[] headers = new Header[0];
private boolean followRedirects = true;
//private boolean ignoreCookies = true;
/**
* creates a new JakartaCommonsHttpClient with given timeout using global remoteProxyConfig
*
* @param timeout in milliseconds
*/
public Client(final int timeout) {
this(timeout, null);
}
/**
* creates a new JakartaCommonsHttpClient with given timeout and requestHeader using global remoteProxyConfig
*
* @param timeout in milliseconds
* @param header header options to send
*/
public Client(final int timeout, final RequestHeader header) {
super();
setTimeout(timeout);
setHeader(header);
}
/*
* (non-Javadoc)
* @see de.anomic.http.HttpClient#setHeader(de.anomic.http.httpHeader)
*/
public void setHeader(final RequestHeader header) {
headers = convertHeaders(header);
}
/*
* (non-Javadoc)
* @see de.anomic.http.HttpClient#setTimeout(int)
*/
public void setTimeout(final int timeout) {
apacheHttpClient.getParams().setIntParameter(HttpMethodParams.SO_TIMEOUT, timeout);
apacheHttpClient.getParams().setIntParameter(HttpMethodParams.HEAD_BODY_CHECK_TIMEOUT, timeout);
conManager.getParams().setConnectionTimeout(timeout);
conManager.getParams().setSoTimeout(timeout);
}
/**
* should redirects automatically be followed?
*
* @param follow
*/
public void setFollowRedirects(final boolean follow) {
followRedirects = follow;
}
/*
* (non-Javadoc)
* @see de.anomic.http.HttpClient#getUserAgent()
*/
public String getUserAgent() {
return getCurrentUserAgent();
}
/**
* This method GETs a page from the server.
*
* @param uri The URI to the page which should be GET.
* @return InputStream of content (body)
* @throws IOException
*/
public ResponseContainer GET(final String uri) throws IOException {
return GET(uri, Long.MAX_VALUE);
}
/**
* This method GETs a page from the server.
*
* @param uri The URI to the page which should be GET.
* @param maxfilesize the maximum allowed filesize (else IOException)
* @return InputStream of content (body)
* @throws IOException
*/
public ResponseContainer GET(final String uri, long maxfilesize) throws IOException {
final HttpMethod get = new ClientGetMethod(uri, maxfilesize);
get.setFollowRedirects(followRedirects);
get.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
return execute(get);
}
public ResponseContainer GET(final String uri, long maxfilesize, String realm) throws IOException {
final HttpMethod get = new ClientGetMethod(uri, maxfilesize);
get.setFollowRedirects(followRedirects);
get.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
get.setRequestHeader("Authorization", "realm=" + realm);
return execute(get);
}
/**
* This method gets only the header of a page.
*
* @param uri The URI to the page whose header should be get.
* @return Instance of response with the content.
* @throws IOException
*/
public ResponseContainer HEAD(final String uri) throws IOException {
assert uri != null : "precondition violated: uri != null";
final HttpMethod head = new HeadMethod(uri);
head.setFollowRedirects(followRedirects);
head.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
return execute(head);
}
/**
* This method POSTs some data from an InputStream to a page.
*
* This is for compatibility (an InputStream does not need to contain correct HTTP!)
*
* @param uri The URI to the page which the post is sent to.
* @param ins InputStream with the data to be posted to the server.
* @return Instance of response with the content.
* @throws IOException
*/
public ResponseContainer POST(final String uri, final InputStream ins) throws IOException {
assert uri != null : "precondition violated: uri != null";
assert ins != null : "precondition violated: ins != null";
final PostMethod post = new PostMethod(uri);
post.setRequestEntity(new InputStreamRequestEntity(ins));
// redirects in POST cause a "Entity enclosing requests cannot be redirected without user intervention" -
// exception
post.setFollowRedirects(false);
return execute(post);
}
/**
* This method sends several data at once via a POST request (multipart-message), maybe compressed
*
* @param uri The URI to the page which the post is sent to.
* @param multiparts {@link java.util.List} with the {@link Part}s of data
* @param gzipBody should the body be compressed
* @return Instance of response with the content.
* @throws IOException
*/
public ResponseContainer POST(final String uri, final List<Part> multiparts, final boolean gzipBody)
throws IOException {
assert uri != null : "precondition violated: uri != null";
final PostMethod post = new PostMethod(uri);
final Part[] parts;
if (multiparts != null) {
parts = multiparts.toArray(new Part[multiparts.size()]);
} else {
// nothing to POST
parts = new Part[0];
}
RequestEntity data = new MultipartRequestEntity(parts, post.getParams());
/*
if (gzipBody) {
data = zipRequest(data);
post.setRequestHeader(HeaderFramework.CONTENT_ENCODING, HeaderFramework.CONTENT_ENCODING_GZIP);
post.setContentChunked(true);
}
*/
post.setRequestEntity(data);
// redirects in POST cause a "Entity enclosing requests cannot be redirected without user intervention" -
// exception
post.setFollowRedirects(false);
return execute(post);
}
/**
* <p>stores the data of the request in a new ByteArrayRequestEntity</p>
*
* <p>when the request is send make sure to set content-encoding-header to gzip!</p>
*
* @param data
* @return a ByteArrayRequestEntitiy with gzipped data
* @throws IOException
*/
/*
private RequestEntity zipRequest(final RequestEntity data) throws IOException {
// cache data and gzip it
final ByteArrayOutputStream zippedBytes = new ByteArrayOutputStream(512);
final GZIPOutputStream toZip = new GZIPOutputStream(zippedBytes);
data.writeRequest(toZip);
toZip.finish();
toZip.flush();
// use compressed data as body (not setting content length according to RFC 2616 HTTP/1.1, section 4.4)
return new ByteArrayRequestEntity(zippedBytes.toByteArray(), data.getContentType());
}
*/
/*
* (non-Javadoc)
* @see de.anomic.http.HttpClient#CONNECT(java.lang.String, int, de.anomic.http.httpHeader)
*/
public ResponseContainer CONNECT(final String host, final int port) throws IOException {
final HostConfiguration hostConfig = new HostConfiguration();
hostConfig.setHost(host, port);
final HttpMethod connect = new ConnectMethod(hostConfig);
connect.setFollowRedirects(false); // there are no redirects possible for CONNECT commands as far as I know.
connect.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
return execute(connect);
}
/**
* adds the yacy-header to the method
*
* @param requestHeader
* @param method
*/
public void addHeader(final RequestHeader requestHeader, final HttpMethod method) {
assert method != null : "precondition violated: method != null";
if (requestHeader != null) {
addHeaders(convertHeaders(requestHeader), method);
}
}
/**
* adds every Header in the array to the method
*
* @param requestHeaders
* @param method must not be null
*/
private static void addHeaders(final Header[] requestHeaders, final HttpMethod method) {
if (method == null) {
throw new NullPointerException("method not set");
}
if (requestHeaders != null) {
for (final Header header : requestHeaders) {
method.addRequestHeader(header);
}
}
}
/**
* convert from yacy-header to apache.commons.httpclient.Header
*
* @param requestHeader
* @return
*/
private static Header[] convertHeaders(final RequestHeader requestHeader) {
final Header[] headers;
if (requestHeader == null) {
headers = new Header[0];
} else {
headers = new Header[requestHeader.size()];
int i = 0;
for (final Entry<String, String> header : requestHeader.entrySet()) {
headers[i] = new Header(header.getKey(), header.getValue());
i++;
}
}
return headers;
}
/**
* executes a method
*
* @param method
* @return
* @throws IOException
*/
private ResponseContainer execute(final HttpMethod method) throws IOException {
assert method != null : "precondition violated: method != null";
setHeader(method);
setProxy(method);
final HostConfiguration hostConfig = ProxySettings.getProxyHostConfig(apacheHttpClient);
// statistics
ConnectionInfo.addConnection(generateConInfo(method));
// execute (send request)
try {
if (hostConfig == null) {
apacheHttpClient.executeMethod(method);
} else {
apacheHttpClient.executeMethod(hostConfig, method);
}
} catch (final IllegalThreadStateException e) {
// cleanUp statistics
MultiProtocolURI url = new MultiProtocolURI(method.getURI().toString());
Latency.slowdown(url);
ConnectionInfo.removeConnection(generateConInfo(method));
throw new IOException(e.getMessage());
} catch (final IOException e) {
// cleanUp statistics
MultiProtocolURI url = new MultiProtocolURI(method.getURI().toString());
Latency.slowdown(url);
ConnectionInfo.removeConnection(generateConInfo(method));
throw e;
} catch (final IllegalStateException e) {
// cleanUp statistics
MultiProtocolURI url = new MultiProtocolURI(method.getURI().toString());
Latency.slowdown(url);
ConnectionInfo.removeConnection(generateConInfo(method));
throw new IOException(e.getMessage());
} finally {
// cleanUp statistics
ConnectionInfo.removeConnection(generateConInfo(method));
}
// return response
return new ResponseContainer(method);
}
/**
* @param method
* @return
* @throws URIException
*/
private void setProxy(final HttpMethod method) throws URIException {
String host = null;
// set proxy
host = method.getURI().getHost();
if (ProxySettings.useForHost(host)) {
final String scheme = method.getURI().getScheme();
if(scheme != null && scheme.toLowerCase().startsWith("https") && !ProxySettings.use4ssl) {
// do not use proxy for HTTPS
return;
}
addProxyAuth(method);
}
}
/**
* @param method
*/
private void setHeader(final HttpMethod method) {
// set header
for (final Header header : headers) {
method.setRequestHeader(header);
}
}
/**
* @param method
* @return
*/
private ConnectionInfo generateConInfo(final HttpMethod method) {
int port = 80;
String host = null;
String protocol = null;
try {
port = method.getURI().getPort();
host = method.getURI().getHost();
protocol = method.getURI().getScheme();
} catch (final URIException e) {
// should not happen, because method is already executed
}
final String query = method.getQueryString() != null ? "?" + method.getQueryString() : "";
return new ConnectionInfo(protocol, port == -1 || port == 80 ? host : host + ":" + port, method.getName() +
" " + method.getPath() + query, method.hashCode(), System.currentTimeMillis());
}
/**
* if necessary adds a header for proxy-authentication
*
* @param method
* @param hostProxyConfig
*/
private void addProxyAuth(final HttpMethod method) {
if (ProxySettings.use) {
final String remoteProxyUser = ProxySettings.user;
if (remoteProxyUser != null && remoteProxyUser.length() > 0) {
assert !remoteProxyUser.contains(":"): "Proxy authentication contains invalid characters: " + remoteProxyUser;
final String remoteProxyPwd = ProxySettings.password;
final String credentials = Base64Order.standardCoder.encodeString(remoteProxyUser.replace(":", "") + ":" + remoteProxyPwd);
method.setRequestHeader(RequestHeader.PROXY_AUTHORIZATION, "Basic " + credentials);
}
}
}
/**
* close all connections
*/
public static void closeAllConnections() {
conManager.closeIdleConnections(1);
conManager.shutdown();
}
/**
* gets the maximum number of connections allowed
*
* @return
*/
public static int maxConnections() {
return conManager.getParams().getMaxTotalConnections();
}
/**
* test
*
* @param args
*/
public static void main(final String[] args) {
ResponseContainer resp = null;
String url = args[0];
if (!url.toUpperCase().startsWith("HTTP://")) {
url = "http://" + url;
}
try {
if (args.length > 1 && "post".equals(args[1])) {
// POST
final ArrayList<Part> files = new ArrayList<Part>();
files.add(new DefaultCharsetFilePart("myfile.txt", new ByteArrayPartSource("myfile.txt", "this is not a file ;)"
.getBytes())));
files.add(new FilePart("anotherfile.raw", new ByteArrayPartSource("anotherfile.raw",
"this is not a binary file ;)".getBytes())));
System.out.println("POST " + files.size() + " elements to " + url);
final Client client = new Client(1000);
resp = client.POST(url, files, false);
System.out.println("----- Header: -----");
System.out.println(resp.getResponseHeader().toString());
System.out.println("----- Body: -----");
System.out.println(new String(resp.getData()));
} else if (args.length > 1 && "head".equals(args[1])) {
// whead
System.out.println("whead " + url);
System.out.println("--------------------------------------");
System.out.println(whead(url).toString());
} else {
// wget
System.out.println("wget " + url);
System.out.println("--------------------------------------");
System.out.println(new String(wget(url, null, 10000)));
}
} catch (final IOException e) {
e.printStackTrace();
} finally {
if (resp != null) {
// release connection
resp.closeStream();
}
}
}
/**
* @return
*/
public static String getCurrentUserAgent() {
return (String) apacheHttpClient.getParams().getParameter(HttpMethodParams.USER_AGENT);
}
/**
* @param userAgent
*/
public static void setUserAgent(final String userAgent) {
apacheHttpClient.getParams().setParameter(HttpMethodParams.USER_AGENT, userAgent + jakartaUserAgent);
}
/**
* remove unused connections
*/
public static void cleanup() {
// do it only once a while
final long now = System.currentTimeMillis();
if (now - lastCleanup > cleanupIntervall) {
lastCleanup = now;
conManager.closeIdleConnections(closeConnectionsAfterMillis);
conManager.deleteClosedConnections();
ConnectionInfo.cleanUp();
}
}
/**
* number of active connections
*
* @return
*/
public static int connectionCount() {
return conManager.getConnectionsInPool();
}
/**
* provide system information for client identification
*/
private static final String systemOST = System.getProperty("os.arch", "no-os-arch") + " " +
System.getProperty("os.name", "no-os-name") + " " + System.getProperty("os.version", "no-os-version") +
"; " + "java " + System.getProperty("java.version", "no-java-version") + "; " + generateLocation();
/**
* generating the location string
*
* @return
*/
public static String generateLocation() {
String loc = System.getProperty("user.timezone", "nowhere");
final int p = loc.indexOf('/');
if (p > 0) {
loc = loc.substring(0, p);
}
loc = loc + "/" + System.getProperty("user.language", "dumb");
return loc;
}
/**
* @return the systemOST
*/
public static String getSystemOST() {
return systemOST;
}
/**
* Gets a page (as raw bytes) addressing vhost at host in uri with specified header and timeout
*
* @param uri
* @param header
* @param vhost
* @param timeout in milliseconds
* @return
* @throws IOException
*/
public static byte[] wget(final String uri) throws IOException {
return wget(uri, new RequestHeader(), 10000, null);
}
public static byte[] wget(final String uri, final RequestHeader header, final int timeout) throws IOException {
return wget(uri, header, timeout, null);
}
public static byte[] wget(final String uri, final RequestHeader header, final int timeout, final String vhost) throws IOException {
assert uri != null : "precondition violated: uri != null";
addHostHeader(header, vhost);
final Client client = new Client(timeout, header);
// do the request
ResponseContainer response = client.GET(uri);
byte[] data = response.getData();
response.closeStream();
return data;
}
/**
* adds a Host-header to the header if vhost is not null
*
* @param header
* @param vhost
* @return
*/
private static void addHostHeader(RequestHeader header, final String vhost) {
if (vhost != null) {
if (header != null) {
header = new RequestHeader();
}
// set host-header
header.add(HeaderFramework.HOST, vhost);
}
}
/**
* Gets a page-header
*
* @param uri
* @return
* @throws IOException
*/
public static ResponseHeader whead(final String uri) throws IOException {
return whead(uri, null);
}
/**
* Gets a page-header
*
* @param uri
* @param header request header
* @return null on error
*/
public static ResponseHeader whead(final String uri, final RequestHeader header) throws IOException {
final Client client = new Client(10000, header);
ResponseContainer response = client.HEAD(uri);
ResponseHeader rh = response.getResponseHeader();
response.closeStream();
return rh;
}
}

67
source/de/anomic/http/client/ClientGetMethod.java
Unescape View File

@ -1,67 +0,0 @@
// httpClientGetMethod.java
// (C) 2009 by David Wieditz; lotus@users.berlios.de
// first published 13.7.2009 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http.client;
import java.io.IOException;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpConnection;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpState;
import org.apache.commons.httpclient.methods.GetMethod;
/**
* this class implements the ability for a maxfilesize
* @author lotus
*
*/
public class ClientGetMethod extends GetMethod {
private long maxfilesize = Long.MAX_VALUE;
public ClientGetMethod(String uri, long maxfilesize) {
super(uri);
if (maxfilesize > 0) this.maxfilesize = maxfilesize;
}
@Override
protected void readResponseHeaders(HttpState state, HttpConnection conn) throws IOException, HttpException {
super.readResponseHeaders(state, conn);
if (this.maxfilesize < Long.MAX_VALUE) {
// already processing the header to be able to throw an exception
Header contentlengthHeader = getResponseHeader("content-length");
long contentlength = 0;
if (contentlengthHeader != null) {
try { contentlength = Long.parseLong(contentlengthHeader.getValue()); } catch (NumberFormatException e) { }
}
if (contentlength > maxfilesize) {
throw new IOException("Content-Length " + contentlength + " larger than maxfilesize " + maxfilesize);
}
}
}
}

220
source/de/anomic/http/client/ConnectionInfo.java
Unescape View File

@ -1,220 +0,0 @@
// HttpConnectionInfo.java
// (C) 2008 by Daniel Raap; danielr@users.berlios.de
// first published 07.04.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http.client;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import net.yacy.kelondro.logging.Log;
/**
* Information about a connection
*
* @author daniel
* @since 07.04.2008
*/
public class ConnectionInfo {
/**
* a list of all current connections
*/
private final static Set<ConnectionInfo> allConnections = Collections
.synchronizedSet(new HashSet<ConnectionInfo>());
// this is only for statistics, so it can be bigger to see lost connectionInfos
private final static int staleAfterMillis = 30 * 60000; // 30 minutes
private final String protocol;
private final String targetHost;
private final String command;
private final int id;
private final long initTime;
/**
* constructor setting all data
*
* @param protocol
* @param targetHost
* @param command
* @param id
* @param initTime
*/
public ConnectionInfo(final String protocol, final String targetHost, final String command, final int id,
final long initTime) {
this.protocol = protocol;
this.targetHost = targetHost;
this.command = command;
this.id = id;
this.initTime = initTime;
}
/**
* @return
*/
public String getProtocol() {
return protocol;
}
/**
* @return
*/
public long getLifetime() {
return System.currentTimeMillis() - initTime;
}
/**
* @return
*/
public int getIdletime() {
return 0;
}
/**
* @return
*/
public String getCommand() {
return command;
}
/**
* @return
*/
public String getTargetHost() {
return targetHost;
}
/**
* @return
*/
public int getID() {
return id;
}
/**
* gets a {@link Set} of all collected ConnectionInfos
*
* Important: iterations must be synchronized!
*
* @return the allConnections
*/
public static Set<ConnectionInfo> getAllConnections() {
return allConnections;
}
/**
* add a connection to the list of all current connections
*
* @param conInfo
*/
public static void addConnection(final ConnectionInfo conInfo) {
allConnections.add(conInfo);
}
/**
* remove a connection from the list of all current connections
*
* @param conInfo
*/
public static void removeConnection(final ConnectionInfo conInfo) {
allConnections.remove(conInfo);
}
/**
* connections with same id {@link equals()} another
*
* @param id
*/
public static void removeConnection(final int id) {
removeConnection(new ConnectionInfo(null, null, null, id, 0));
}
/**
* removes stale connections
*/
public static void cleanUp() {
try {
synchronized (allConnections) {
final int sizeBefore = allConnections.size();
for(final ConnectionInfo con: allConnections) {
if(con.getLifetime() > staleAfterMillis) {
allConnections.remove(con);
}
}
if (Log.isFine("HTTPC")) Log.logFine("HTTPC", "cleanUp ConnectionInfo removed "+ (sizeBefore - allConnections.size()));
}
} catch (final java.util.ConcurrentModificationException e) {
Log.logWarning("HTTPC", "cleanUp ConnectionInfo interrupted by ConcurrentModificationException");
}
}
/* (non-Javadoc)
* @see java.lang.Object#toString()
*/
@Override
public String toString() {
final StringBuilder string = new StringBuilder(50);
string.append("ID ");
string.append(getID());
string.append(", ");
string.append(getProtocol());
string.append("://");
string.append(getTargetHost());
string.append(" ");
string.append(getCommand());
string.append(", since ");
string.append(getLifetime());
string.append(" ms");
return string.toString();
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#hashCode()
*/
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + id;
return result;
}
/*
* (non-Javadoc)
*
* @see java.lang.Object#equals(java.lang.Object)
*/
@Override
public boolean equals(final Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
final ConnectionInfo other = (ConnectionInfo) obj;
return this.id == other.id;
}
}

49
source/de/anomic/http/client/DefaultCharsetFilePart.java
Unescape View File

@ -1,49 +0,0 @@
// DefaultCharsetFilePart.java
// (C) 2008 by Daniel Raap; danielr@users.berlios.de
// first published 02.08.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http.client;
import java.nio.charset.Charset;
import org.apache.commons.httpclient.methods.multipart.FilePart;
import org.apache.commons.httpclient.methods.multipart.PartSource;
/**
* @author daniel
*
*/
public class DefaultCharsetFilePart extends FilePart {
/**
* Konstruktor
* @param name
* @param partSource
*/
public DefaultCharsetFilePart(String name, PartSource partSource) {
super(name, partSource, null, Charset.defaultCharset().name());
}
}

50
source/de/anomic/http/client/DefaultCharsetStringPart.java
Unescape View File

@ -1,50 +0,0 @@
// DefaultCharsetStringPart.java
// (C) 2008 by Daniel Raap; danielr@users.berlios.de
// first published 02.08.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http.client;
import java.nio.charset.Charset;
import org.apache.commons.httpclient.methods.multipart.StringPart;
/**
* @author daniel
*
*/
public class DefaultCharsetStringPart extends StringPart {
/**
* constructs a StringPart with the defaultCharset (instead of US-ASCII)
*
* @see Charset#defaultCharset()
* @param name
* @param value
*/
public DefaultCharsetStringPart(String name, String value) {
super(name, value, Charset.defaultCharset().name());
}
}

236
source/de/anomic/http/client/EasySSLProtocolSocketFactory.java
Unescape View File

@ -1,236 +0,0 @@
/*
* $Header: /home/jerenkrantz/tmp/commons/commons-convert/cvs/home/cvs/jakarta-commons//httpclient/src/contrib/org/apache/commons/httpclient/contrib/ssl/EasySSLProtocolSocketFactory.java,v 1.7 2004/06/11 19:26:27 olegk Exp $
* $Revision: 480424 $
* $Date: 2006-11-29 06:56:49 +0100 (Wed, 29 Nov 2006) $
*
* ====================================================================
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
/* CHANGES to original file
* ========================
* 2008-04-07 danielr: changed package from org.apache.commons.httpclient.contrib.ssl
*/
package de.anomic.http.client;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.UnknownHostException;
import javax.net.SocketFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import org.apache.commons.httpclient.ConnectTimeoutException;
import org.apache.commons.httpclient.HttpClientError;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* <p>
* EasySSLProtocolSocketFactory can be used to creats SSL {@link Socket}s
* that accept self-signed certificates.
* </p>
* <p>
* This socket factory SHOULD NOT be used for productive systems
* due to security reasons, unless it is a concious decision and
* you are perfectly aware of security implications of accepting
* self-signed certificates
* </p>
*
* <p>
* Example of using custom protocol socket factory for a specific host:
* <pre>
* Protocol easyhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
*
* HttpClient client = new HttpClient();
* client.getHostConfiguration().setHost("localhost", 443, easyhttps);
* // use relative url only
* GetMethod httpget = new GetMethod("/");
* client.executeMethod(httpget);
* </pre>
* </p>
* <p>
* Example of using custom protocol socket factory per default instead of the standard one:
* <pre>
* Protocol easyhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
* Protocol.registerProtocol("https", easyhttps);
*
* HttpClient client = new HttpClient();
* GetMethod httpget = new GetMethod("https://localhost/");
* client.executeMethod(httpget);
* </pre>
* </p>
*
* @author <a href="mailto:oleg -at- ural.ru">Oleg Kalnichevski</a>
*
* <p>
* DISCLAIMER: HttpClient developers DO NOT actively support this component.
* The component is provided as a reference material, which may be inappropriate
* for use without additional customization.
* </p>
*/
public class EasySSLProtocolSocketFactory implements SecureProtocolSocketFactory {
/** Log object for this class. */
private static final Log LOG = LogFactory.getLog(EasySSLProtocolSocketFactory.class);
private SSLContext sslcontext = null;
/**
* Constructor for EasySSLProtocolSocketFactory.
*/
public EasySSLProtocolSocketFactory() {
super();
}
private static SSLContext createEasySSLContext() {
try {
final SSLContext context = SSLContext.getInstance("SSL");
context.init(
null,
new TrustManager[] {new EasyX509TrustManager(null)},
null);
return context;
} catch (final Exception e) {
LOG.error(e.getMessage(), e);
throw new HttpClientError(e.toString());
}
}
private SSLContext getSSLContext() {
if (this.sslcontext == null) {
this.sslcontext = createEasySSLContext();
}
return this.sslcontext;
}
/**
* @see SecureProtocolSocketFactory#createSocket(java.lang.String,int,java.net.InetAddress,int)
*/
public Socket createSocket(
final String host,
final int port,
final InetAddress clientHost,
final int clientPort)
throws IOException, UnknownHostException {
return getSSLContext().getSocketFactory().createSocket(
host,
port,
clientHost,
clientPort
);
}
/**
* Attempts to get a new socket connection to the given host within the given time limit.
* <p>
* To circumvent the limitations of older JREs that do not support connect timeout a
* controller thread is executed. The controller thread attempts to create a new socket
* within the given limit of time. If socket constructor does not return until the
* timeout expires, the controller terminates and throws an {@link ConnectTimeoutException}
* </p>
*
* @param host the host name/IP
* @param port the port on the host
* @param clientHost the local host name/IP to bind the socket to
* @param clientPort the port on the local machine
* @param params {@link HttpConnectionParams Http connection parameters}
*
* @return Socket a new socket
*
* @throws IOException if an I/O error occurs while creating the socket
* @throws UnknownHostException if the IP address of the host cannot be
* determined
*/
public Socket createSocket(
final String host,
final int port,
final InetAddress localAddress,
final int localPort,
final HttpConnectionParams params
) throws IOException, UnknownHostException, ConnectTimeoutException {
if (params == null) {
throw new IllegalArgumentException("Parameters may not be null");
}
final int timeout = params.getConnectionTimeout();
final SocketFactory socketfactory = getSSLContext().getSocketFactory();
final Socket socket;
if (timeout == 0) {
socket = socketfactory.createSocket(host, port, localAddress, localPort);
} else {
socket = socketfactory.createSocket();
final SocketAddress localaddr = new InetSocketAddress(localAddress, localPort);
final SocketAddress remoteaddr = new InetSocketAddress(host, port);
socket.bind(localaddr);
socket.connect(remoteaddr, timeout);
}
return socket;
}
/**
* @see SecureProtocolSocketFactory#createSocket(java.lang.String,int)
*/
public Socket createSocket(final String host, final int port)
throws IOException, UnknownHostException {
return getSSLContext().getSocketFactory().createSocket(
host,
port
);
}
/**
* @see SecureProtocolSocketFactory#createSocket(java.net.Socket,java.lang.String,int,boolean)
*/
public Socket createSocket(
final Socket socket,
final String host,
final int port,
final boolean autoClose)
throws IOException, UnknownHostException {
return getSSLContext().getSocketFactory().createSocket(
socket,
host,
port,
autoClose
);
}
public boolean equals(final Object obj) {
return ((obj != null) && obj.getClass().equals(this.getClass()));
}
public int hashCode() {
return EasySSLProtocolSocketFactory.class.hashCode();
}
}

119
source/de/anomic/http/client/EasyX509TrustManager.java
Unescape View File

@ -1,119 +0,0 @@
/*
* ====================================================================
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
/* CHANGES to original file
* ========================
* 2008-04-07 danielr: changed package from org.apache.commons.httpclient.contrib.ssl
*/
package de.anomic.http.client;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* <p>
* EasyX509TrustManager unlike default {@link X509TrustManager} accepts
* self-signed certificates.
* </p>
* <p>
* This trust manager SHOULD NOT be used for productive systems
* due to security reasons, unless it is a concious decision and
* you are perfectly aware of security implications of accepting
* self-signed certificates
* </p>
*
* @author <a href="mailto:adrian.sutton@ephox.com">Adrian Sutton</a>
* @author <a href="mailto:oleg@ural.ru">Oleg Kalnichevski</a>
*
* <p>
* DISCLAIMER: HttpClient developers DO NOT actively support this component.
* The component is provided as a reference material, which may be inappropriate
* for use without additional customization.
* </p>
*/
public class EasyX509TrustManager implements X509TrustManager
{
private X509TrustManager standardTrustManager = null;
/** Log object for this class. */
private static final Log LOG = LogFactory.getLog(EasyX509TrustManager.class);
/**
* Constructor for EasyX509TrustManager.
*/
public EasyX509TrustManager(final KeyStore keystore) throws NoSuchAlgorithmException, KeyStoreException {
super();
final TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
factory.init(keystore);
final TrustManager[] trustmanagers = factory.getTrustManagers();
if (trustmanagers.length == 0) {
throw new NoSuchAlgorithmException("no trust manager found");
}
this.standardTrustManager = (X509TrustManager)trustmanagers[0];
}
/**
* @see javax.net.ssl.X509TrustManager#checkClientTrusted(X509Certificate[],String authType)
*/
public void checkClientTrusted(final X509Certificate[] certificates,final String authType) throws CertificateException {
standardTrustManager.checkClientTrusted(certificates,authType);
}
/**
* @see javax.net.ssl.X509TrustManager#checkServerTrusted(X509Certificate[],String authType)
*/
public void checkServerTrusted(final X509Certificate[] certificates,final String authType) throws CertificateException {
if ((certificates != null) && LOG.isDebugEnabled()) {
LOG.debug("Server certificate chain:");
for (int i = 0; i < certificates.length; i++) {
LOG.debug("X509Certificate[" + i + "]=" + certificates[i]);
}
}
if ((certificates != null) && (certificates.length == 1)) {
certificates[0].checkValidity();
} else {
standardTrustManager.checkServerTrusted(certificates,authType);
}
}
/**
* @see javax.net.ssl.X509TrustManager#getAcceptedIssuers()
*/
public X509Certificate[] getAcceptedIssuers() {
return this.standardTrustManager.getAcceptedIssuers();
}
}

1
source/de/anomic/http/server/HTTPDProxyHandler.java
Unescape View File

@ -87,7 +87,6 @@ import net.yacy.repository.Blacklist;
import de.anomic.crawler.retrieval.HTTPLoader;
import de.anomic.crawler.retrieval.Request;
import de.anomic.crawler.retrieval.Response;
import de.anomic.http.client.MultiOutputStream;
//import de.anomic.http.client.Client;
import de.anomic.http.client.Cache;
import de.anomic.search.Switchboard;

2
source/de/anomic/http/client/MultiOutputStream.java → source/de/anomic/http/server/MultiOutputStream.java
Unescape View File

@ -2,7 +2,7 @@
* MultiOutputStream.java
* @since 26.08.2008
*/
package de.anomic.http.client;
package de.anomic.http.server;
import java.io.IOException;
import java.io.OutputStream;

179
source/de/anomic/http/server/ResponseContainer.java
Unescape View File

@ -1,179 +0,0 @@
// ResponseContainer.java
// (C) 2008 by Daniel Raap; danielr@users.berlios.de
// first published 2.4.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http.server;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import net.yacy.kelondro.io.ByteCountInputStream;
import net.yacy.kelondro.util.FileUtils;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpMethod;
import de.anomic.http.client.ConnectionInfo;
/**
* container for http-response data
*
* @author daniel
* @since 21.03.2008
*/
public class ResponseContainer {
private final HttpMethod method;
private String incomingAccountingName = null;
/**
* cache of body-data
*/
private byte[] responseBody;
/**
* constructor
*
* @param method
* @throws IOException
*/
public ResponseContainer(final HttpMethod method) {
super();
this.method = method;
}
/*
* (non-Javadoc)
*
* @see de.anomic.http.HttpResponse#getResponseHeader()
*/
public ResponseHeader getResponseHeader() {
final ResponseHeader responseHeader = new ResponseHeader();
for (final Header header : method.getResponseHeaders()) {
responseHeader.add(header.getName(), header.getValue());
}
return responseHeader;
}
/**
* @see org.apache.commons.httpclient.HttpMethod#getResponseBody()
* @return
* @throws IOException
*/
public byte[] getData() throws IOException {
if (responseBody == null) {
InputStream instream = null;
try {
instream = getDataAsStream();
if (instream != null) {
responseBody = FileUtils.read(instream);
}
} finally {
if (instream != null) {
closeStream();
}
}
}
return responseBody;
}
/**
* @see org.apache.commons.httpclient.HttpMethod#getResponseBodyAsStream()
* @return
* @throws IOException
*/
public InputStream getDataAsStream() throws IOException {
InputStream inStream = method.getResponseBodyAsStream();
if (inStream == null) {
return null;
}
if (getResponseHeader().gzip()) {
inStream = new GZIPInputStream(inStream);
}
// count bytes for overall http-statistics
return new ByteCountInputStream(inStream, incomingAccountingName);
}
/**
* Abort Download, e. g. when proxy connection was closed
* Do not forget to call closeStream afterwards...
*
*/
public void abort() {
method.abort();
}
/*
* (non-Javadoc)
*
* @see de.anomic.http.HttpResponse#closeStream()
*/
public void closeStream() {
method.releaseConnection();
// statistics
ConnectionInfo.removeConnection(method.hashCode());
}
/*
* (non-Javadoc)
*
* @see de.anomic.http.HttpResponse#getStatusLine()
*/
public String getStatusLine() {
return getStatusCode() + " " + method.getStatusText();
}
/*
* (non-Javadoc)
*
* @see de.anomic.http.HttpResponse#getStatusCode()
*/
public int getStatusCode() {
final int code = method.getStatusCode();
assert code >= 100 && code <= 999 : "postcondition violated: StatusCode ("
+ code + ") has not 3 digits!";
return code;
}
/*
* (non-Javadoc)
*
* @see de.anomic.http.HttpResponse#getHttpVer()
*/
public String getHttpVer() {
return method.getStatusLine().getHttpVersion();
}
/**
* sets the name for accounting incoming (loaded) bytes
*
* @param accName
*/
public void setAccountingName(final String accName) {
incomingAccountingName = accName;
}
}
Write Preview
Loading…
Cancel
Save