boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed System.err and System.out usage from FTPClient; changed logging to log4j (preferred in yacy.cora)

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7064 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 90531f78ff
commit caece04f26
7 changed files with 260 additions and 1262 deletions
Show Stats Download Patch File Download Diff File

3
source/de/anomic/crawler/retrieval/FTPLoader.java
Unescape View File

@ -94,8 +94,7 @@ public class FTPLoader {
final ByteArrayOutputStream berr = new ByteArrayOutputStream();
// create new ftp client
final PrintStream err = new PrintStream(berr);
final FTPClient ftpClient = new FTPClient(System.in, null, err);
final FTPClient ftpClient = new FTPClient();
ftpClient.setDataTimeoutByMaxFilesize(maxFileSize);
// get a connection

644
source/net/yacy/cora/protocol/Client.java
Unescape View File

@ -1,644 +0,0 @@
/**
* Client
* Copyright 2010 by Sebastian Gaebel
* First released 01.07.2010 at http://yacy.net
*
* $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
* $LastChangedRevision: 7020 $
* $LastChangedBy: sixcooler $
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.protocol;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.util.LinkedHashMap;
import java.util.Set;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.message.BasicHeader;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
/**
* HttpClient implementation which uses HttpComponents Client {@link http://hc.apache.org/}
*
* @author sixcooler
*
*/
public class Client {
private final static int maxcon = 20;
private static IdledConnectionEvictor idledConnectionEvictor = null;
private static HttpClient httpClient = null;
private Header[] headers = null;
private HttpResponse httpResponse = null;
private HttpUriRequest currentRequest = null;
private long upbytes = 0L;
private int timeout = 10000;
private String userAgent = null;
private String host = null;
private boolean redirecting = true;
private String realm = null;
public Client() {
super();
if (httpClient == null) {
initConnectionManager();
}
}
public static void setDefaultUserAgent(final String defaultAgent) {
HttpProtocolParams.setUserAgent(httpClient.getParams(), defaultAgent);
}
private static void initConnectionManager() {
// Create and initialize HTTP parameters
final HttpParams httpParams = new BasicHttpParams();
/**
* ConnectionManager settings
*/
// TODO: how much connections do we need? - default: 20
ConnManagerParams.setMaxTotalConnections(httpParams, maxcon);
// for statistics same value should also be set here
ConnectionInfo.setMaxcount(maxcon);
// connections per host (2 default)
final ConnPerRouteBean connPerRoute = new ConnPerRouteBean(2);
// Increase max connections for localhost
HttpHost localhost = new HttpHost("locahost");
connPerRoute.setMaxForRoute(new HttpRoute(localhost), maxcon);
ConnManagerParams.setMaxConnectionsPerRoute(httpParams, connPerRoute);
// how long to wait for getting a connection from manager in milliseconds
ConnManagerParams.setTimeout(httpParams, 3000L);
/**
* HTTP protocol settings
*/
HttpProtocolParams.setVersion(httpParams, HttpVersion.HTTP_1_1);
// UserAgent
HttpProtocolParams.setUserAgent(httpParams, "yacy (" + systemOST +") yacy.net");
HttpProtocolParams.setUseExpectContinue(httpParams, true);
/**
* HTTP connection settings
*/
// timeout in milliseconds until a connection is established in milliseconds
HttpConnectionParams.setConnectionTimeout(httpParams, 10000);
// SO_LINGER affects the socket close operation in seconds
// HttpConnectionParams.setLinger(httpParams, 6);
// TODO: is default ok?
// HttpConnectionParams.setSocketBufferSize(httpParams, 8192);
// SO_TIMEOUT: maximum period inactivity between two consecutive data packets in milliseconds
HttpConnectionParams.setSoTimeout(httpParams, 5000);
// getting an I/O error when executing a request over a connection that has been closed at the server side
HttpConnectionParams.setStaleCheckingEnabled(httpParams, true);
// conserve bandwidth by minimizing the number of segments that are sent
HttpConnectionParams.setTcpNoDelay(httpParams, false);
// TODO: testing noreuse - there will be HttpConnectionParams.setSoReuseaddr(HttpParams params, boolean reuseaddr) in core-4.1
// Create and initialize scheme registry
final SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443));
ClientConnectionManager clientConnectionManager = new ThreadSafeClientConnManager(httpParams, schemeRegistry);
httpClient = new DefaultHttpClient(clientConnectionManager, httpParams);
// ask for gzip
((AbstractHttpClient) httpClient).addRequestInterceptor(new GzipRequestInterceptor());
// uncompress gzip
((AbstractHttpClient) httpClient).addResponseInterceptor(new GzipResponseInterceptor());
idledConnectionEvictor = new IdledConnectionEvictor(clientConnectionManager);
idledConnectionEvictor.start();
}
/**
* This method should be called just before shutdown
* to stop the ConnectionManager and idledConnectionEvictor
*
* @throws InterruptedException
*/
public static void closeConnectionManager() throws InterruptedException {
if (idledConnectionEvictor != null) {
// Shut down the evictor thread
idledConnectionEvictor.shutdown();
idledConnectionEvictor.join();
}
if (httpClient != null) {
// Shut down the connection manager
httpClient.getConnectionManager().shutdown();
}
}
/**
* This method sets the Header used for the request
*
* @param entrys to be set as request header
*/
public void setHeader(final Set<Entry<String, String>> entrys) {
if (entrys != null) {
int i = 0;
headers = new Header[entrys.size()];
for (final Entry<String, String> entry : entrys) {
headers[i++] = new BasicHeader(entry.getKey(),entry.getValue());
}
}
}
/**
* This method sets the timeout of the Connection and Socket
*
* @param timeout in milliseconds
*/
public void setTimout(final int timeout) {
this.timeout = timeout;
}
/**
* This method sets the UserAgent to be used for the request
*
* @param userAgent
*/
public void setUserAgent(final String userAgent) {
this.userAgent = userAgent;
}
/**
* This method sets the host to be called at the request
*
* @param host
*/
public void setHost(final String host) {
this.host = host;
}
/**
* This method sets if requests should follow redirects
*
* @param redirecting
*/
public void setRedirecting(final boolean redirecting) {
this.redirecting = redirecting;
}
/**
* This method sets the authorization realm for the request
*
* @param realm
*/
public void setRealm(final String realm) {
this.realm = realm;
}
/**
* This method GETs a page from the server.
*
* @param uri the url to get
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final String uri) throws IOException {
return GETbytes(uri, Long.MAX_VALUE);
}
/**
* This method GETs a page from the server.
*
* @param uri the url to get
* @param maxBytes to get
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final String uri, long maxBytes) throws IOException {
final HttpGet httpGet = new HttpGet(uri);
return getContentBytes(httpGet, maxBytes);
}
/**
* This method GETs a page from the server.
* to be used for streaming out
* Please take care to call finish()!
*
* @param uri the url to get
* @throws IOException
*/
public void GET(final String uri) throws IOException {
if (currentRequest != null) throw new IOException("Client is in use!");
final HttpGet httpGet = new HttpGet(uri);
currentRequest = httpGet;
execute(httpGet);
}
/**
* This method gets HEAD response
*
* @param uri the url to Response from
* @return the HttpResponse
* @throws IOException
*/
public HttpResponse HEADResponse(final String uri) throws IOException {
final HttpHead httpHead = new HttpHead(uri);
execute(httpHead);
finish();
ConnectionInfo.removeConnection(httpHead.hashCode());
return httpResponse;
}
/**
* This method POSTs a page from the server.
* to be used for streaming out
* Please take care to call finish()!
*
* @param uri the url to post
* @param instream the input to post
* @param length the contentlength
* @throws IOException
*/
public void POST(final String uri, final InputStream instream, long length) throws IOException {
if (currentRequest != null) throw new IOException("Client is in use!");
final HttpPost httpPost = new HttpPost(uri);
final InputStreamEntity inputStreamEntity = new InputStreamEntity(instream, length);
// statistics
upbytes = length;
httpPost.setEntity(inputStreamEntity);
currentRequest = httpPost;
execute(httpPost);
}
/**
* This method POSTs a page from the server.
*
* @param uri the url to post
* @param parts to post
* @return content bytes
* @throws IOException
*/
public byte[] POSTbytes(final String uri, final LinkedHashMap<String,ContentBody> parts) throws IOException {
final HttpPost httpPost = new HttpPost(uri);
final MultipartEntity multipartEntity = new MultipartEntity();
for (Entry<String,ContentBody> part : parts.entrySet())
multipartEntity.addPart(part.getKey(), part.getValue());
// statistics
upbytes = multipartEntity.getContentLength();
httpPost.setEntity(multipartEntity);
return getContentBytes(httpPost, Long.MAX_VALUE);
}
/**
*
* @return HttpResponse from call
*/
public HttpResponse getHttpResponse() {
return httpResponse;
}
/**
*
* @return status code from http request
*/
public int getStatusCode() {
return httpResponse.getStatusLine().getStatusCode();
}
/**
* This method gets direct access to the content-stream
* Since this way is uncontrolled by the Client think of using 'writeTo' instead!
* Please take care to call finish()!
*
* @return the content as InputStream
* @throws IOException
*/
public InputStream getContentstream() throws IOException {
if (httpResponse != null && currentRequest != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) try {
return httpEntity.getContent();
} catch (final IOException e) {
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest.abort();
currentRequest = null;
throw e;
}
}
return null;
}
/**
* This method streams the content to the outputStream
* Please take care to call finish()!
*
* @param outputStream
* @throws IOException
*/
public void writeTo(final OutputStream outputStream) throws IOException {
if (httpResponse != null && currentRequest != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) try {
httpEntity.writeTo(outputStream);
outputStream.flush();
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest = null;
} catch (final IOException e) {
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest.abort();
currentRequest = null;
throw e;
}
}
}
/**
* This method ensures correct finish of client-connections
* This method should be used after every use of GET or POST and writeTo or getContentstream!
*
* @throws IOException
*/
public void finish() throws IOException {
if (httpResponse != null) {
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null && httpEntity.isStreaming()) {
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
}
}
if (currentRequest != null) {
ConnectionInfo.removeConnection(currentRequest.hashCode());
currentRequest.abort();
currentRequest = null;
}
}
private byte[] getContentBytes(final HttpUriRequest httpUriRequest, final long maxBytes) throws IOException {
byte[] content = null;
try {
execute(httpUriRequest);
if (httpResponse == null) return null;
// get the response body
final HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) {
if (httpEntity.getContentLength() < maxBytes) {
content = EntityUtils.toByteArray(httpEntity);
}
// TODO: The name of this method is misnomer.
// It will be renamed to #finish() in the next major release of httpcore
httpEntity.consumeContent();
}
} catch (final IOException e) {
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
httpUriRequest.abort();
throw e;
}
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
return content;
}
private void execute(final HttpUriRequest httpUriRequest) throws IOException {
final HttpContext httpContext = new BasicHttpContext();
setHeaders(httpUriRequest);
setParams(httpUriRequest.getParams());
setProxy(httpUriRequest.getParams());
// statistics
storeConnectionInfo(httpUriRequest);
try {
// execute the method
httpResponse = httpClient.execute(httpUriRequest, httpContext);
} catch (ClientProtocolException e) {
ConnectionInfo.removeConnection(httpUriRequest.hashCode());
httpUriRequest.abort();
throw new IOException("Client can't execute: " + e.getMessage());
}
}
private void setHeaders(final HttpUriRequest httpUriRequest) {
if (headers != null) {
for (Header header : headers) {
httpUriRequest.addHeader(header);
}
}
if (realm != null)
httpUriRequest.setHeader("Authorization", "realm=" + realm);
}
private void setParams(final HttpParams httpParams) {
HttpClientParams.setRedirecting(httpParams, redirecting);
HttpConnectionParams.setConnectionTimeout(httpParams, timeout);
HttpConnectionParams.setSoTimeout(httpParams, timeout);
if (userAgent != null)
HttpProtocolParams.setUserAgent(httpParams, userAgent);
if (host != null)
httpParams.setParameter(HTTP.TARGET_HOST, host);
}
private void setProxy(final HttpParams httpParams) {
if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this
ProxySettings.setProxyCreds((AbstractHttpClient) httpClient);
}
private void storeConnectionInfo(final HttpUriRequest httpUriRequest) {
final int port = httpUriRequest.getURI().getPort();
final String thost = httpUriRequest.getURI().getHost();
ConnectionInfo.addConnection(new ConnectionInfo(
httpUriRequest.getURI().getScheme(),
port == 80 ? thost : thost + ":" + port,
httpUriRequest.getMethod() + " " + httpUriRequest.getURI().getPath(),
httpUriRequest.hashCode(),
System.currentTimeMillis(),
upbytes));
}
/**
* provide system information for client identification
*/
private static final String systemOST = System.getProperty("os.arch", "no-os-arch") + " " +
System.getProperty("os.name", "no-os-name") + " " + System.getProperty("os.version", "no-os-version") +
"; " + "java " + System.getProperty("java.version", "no-java-version") + "; " + generateLocation();
/**
* generating the location string
*
* @return
*/
public static String generateLocation() {
String loc = System.getProperty("user.timezone", "nowhere");
final int p = loc.indexOf('/');
if (p > 0) {
loc = loc.substring(0, p);
}
loc = loc + "/" + System.getProperty("user.language", "dumb");
return loc;
}
/**
* @return the systemOST
*/
public static String getSystemOST() {
return systemOST;
}
/**
* testing
*
* @param args urls to test
*/
public static void main(final String[] args) {
String url = null;
// prepare Parts
final LinkedHashMap<String,ContentBody> newparts = new LinkedHashMap<String,ContentBody>();
try {
newparts.put("foo", new StringBody("FooBar"));
newparts.put("bar", new StringBody("BarFoo"));
} catch (UnsupportedEncodingException e) {
System.out.println(e.getStackTrace());
}
Client client = new Client();
client.setUserAgent("foobar");
client.setRedirecting(false);
// Get some
for (int i = 0; i < args.length; i++) {
url = args[i];
if (!url.toUpperCase().startsWith("HTTP://")) {
url = "http://" + url;
}
try {
System.out.println(new String(client.GETbytes(url)));
} catch (IOException e) {
e.printStackTrace();
}
}
// Head some
// try {
// client.HEADResponse(url);
// } catch (IOException e) {
// e.printStackTrace();
// }
for (Header header: client.getHttpResponse().getAllHeaders()) {
System.out.println("Header " + header.getName() + " : " + header.getValue());
// for (HeaderElement element: header.getElements())
// System.out.println("Element " + element.getName() + " : " + element.getValue());
}
System.out.println(client.getHttpResponse().getLocale());
System.out.println(client.getHttpResponse().getProtocolVersion());
System.out.println(client.getHttpResponse().getStatusLine());
// Post some
// try {
// System.out.println(new String(client.POSTbytes(url, newparts)));
// } catch (IOException e1) {
// e1.printStackTrace();
// }
// Close out connection manager
try {
Client.closeConnectionManager();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
*
* @see: http://hc.apache.org/httpcomponents-client-4.0.1/tutorial/html/connmgmt.html#d4e638
*
*/
public static class IdledConnectionEvictor extends Thread {
private final ClientConnectionManager clientConnectionManager;
private volatile boolean shutdown;
public IdledConnectionEvictor(ClientConnectionManager clientConnectionManager) {
super();
this.clientConnectionManager = clientConnectionManager;
}
@Override
public void run() {
try {
while (!shutdown) {
synchronized (this) {
wait(5000);
// Close expired connections
clientConnectionManager.closeExpiredConnections();
// Optionally, close connections
// that have been idle longer than 5 sec
// (some SOHO router act strange on >5sec idled connections)
clientConnectionManager.closeIdleConnections(5, TimeUnit.SECONDS);
}
}
} catch (InterruptedException ex) {
// terminate
}
}
public void shutdown() {
shutdown = true;
synchronized (this) {
notifyAll();
}
}
}
}

73
source/net/yacy/cora/protocol/GzipDecompressingEntity.java
Unescape View File

@ -1,73 +0,0 @@
/**
* GzipDecompressingEntity
* Copyright 2010 by Sebastian Gaebel
* First released 01.07.2010 at http://yacy.net
*
* $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
* $LastChangedRevision: 7011 $
* $LastChangedBy: sixcooler $
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.protocol;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.GZIPInputStream;
import org.apache.http.HttpEntity;
import org.apache.http.entity.HttpEntityWrapper;
public class GzipDecompressingEntity extends HttpEntityWrapper {
private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size
public GzipDecompressingEntity(final HttpEntity entity) {
super(entity);
}
public InputStream getContent() throws IOException, IllegalStateException {
// the wrapped entity's getContent() decides about repeatability
InputStream wrappedin = wrappedEntity.getContent();
return new GZIPInputStream(wrappedin);
}
public void writeTo(OutputStream outstream) throws IOException {
if (outstream == null) {
throw new IllegalArgumentException("Output stream may not be null");
}
InputStream instream = this.getContent();
int l;
byte[] tmp = new byte[DEFAULT_BUFFER_SIZE];
while ((l = instream.read(tmp)) != -1) {
outstream.write(tmp, 0, l);
}
}
public boolean isChunked() {
return true;
}
public long getContentLength() {
// length of ungzipped content not known in advance
return -1;
}
}

46
source/net/yacy/cora/protocol/GzipRequestInterceptor.java
Unescape View File

@ -1,46 +0,0 @@
/**
* GzipRequestInterceptor
* Copyright 2010 by Sebastian Gaebel
* First released 01.07.2010 at http://yacy.net
*
* $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
* $LastChangedRevision: 7001 $
* $LastChangedBy: sixcooler $
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.protocol;
import java.io.IOException;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.protocol.HttpContext;
public class GzipRequestInterceptor implements HttpRequestInterceptor {
private static final String ACCEPT_ENCODING = "Accept-Encoding";
private static final String GZIP_CODEC = "gzip";
public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
if (!request.containsHeader(ACCEPT_ENCODING)) {
request.addHeader(ACCEPT_ENCODING, GZIP_CODEC);
}
}
}

62
source/net/yacy/cora/protocol/GzipResponseInterceptor.java
Unescape View File

@ -1,62 +0,0 @@
/**
* GzipResponseInterceptor
* Copyright 2010 by Sebastian Gaebel
* First released 01.07.2010 at http://yacy.net
*
* $LastChangedDate: 2010-06-16 17:11:21 +0200 (Mi, 16 Jun 2010) $
* $LastChangedRevision: 7001 $
* $LastChangedBy: sixcooler $
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.protocol;
import java.io.IOException;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.protocol.HttpContext;
public class GzipResponseInterceptor implements HttpResponseInterceptor {
private static final String GZIP_CODEC = "gzip";
public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException {
if (context == null) {
throw new IllegalArgumentException("HTTP context may not be null");
}
HttpEntity entity = response.getEntity();
if (entity != null) {
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
HeaderElement[] codecs = ceheader.getElements();
for (int i = 0; i < codecs.length; i++) {
if (codecs[i].getName().equalsIgnoreCase(GZIP_CODEC)) {
// response.removeHeader(ceheader);
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
}

97
source/net/yacy/cora/protocol/ProxySettings.java
Unescape View File

@ -1,97 +0,0 @@
/**
* ProxySettings
* Copyright 2010 by Michael Peter Christen
* First released 25.05.2010 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.protocol;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.AbstractHttpClient;
/**
* settings for a remote proxy
*
*/
public final class ProxySettings {
// Dummy value to associate with an Object in the backing Map
private static final Object PRESENT = new Object();
public static boolean use = false, use4YaCy = false, use4ssl = false;
public static String host = null, user = "", password = "";
public static int port = 0;
public static String[] noProxy = null;
public static final Map<String, Object> allowProxy = new ConcurrentHashMap<String, Object>();
public static final Map<String, Object> disallowProxy = new ConcurrentHashMap<String, Object>();
/**
* produce a HostConfiguration (apache object) with the proxy access information included
* @param apacheHttpClient
* @return a host configuration with proxy config if the proxy shall be used; a cloned configuration otherwise
*/
public static HostConfiguration getProxyHostConfig(HttpClient apacheHttpClient) {
final HostConfiguration hostConfig;
if (!use) return null;
hostConfig = new HostConfiguration(apacheHttpClient.getHostConfiguration());
hostConfig.setProxy(host, port);
return hostConfig;
}
/**
*
* @return the HttpHost to be used as proxy
*/
public static HttpHost getProxyHost() {
if (!use) return null;
return new HttpHost(host, port);
}
public static void setProxyCreds(AbstractHttpClient httpClient) {
if (!use) return;
httpClient.getCredentialsProvider().setCredentials(
new AuthScope(host, port),
new UsernamePasswordCredentials(user, password));
}
/**
* tell if a remote proxy will be used for the given host
* @param host
* @return true, if the proxy shall be used for the given host
*/
public static boolean useForHost(final String host) {
if (!use) return false;
if (allowProxy.containsKey(host)) return true;
if (disallowProxy.containsKey(host)) return false;
for (String pattern: noProxy) {
if (host.matches(pattern)) {
disallowProxy.put(host, PRESENT);
return false;
}
}
allowProxy.put(host, PRESENT);
return true;
}
}

597
source/net/yacy/cora/protocol/ftp/FTPClient.java
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save