refactoring of HttpClient Writer processes

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4678 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 8aa9fd8f24
commit 202a3adb3e

@ -375,7 +375,7 @@ public class Network {
prop.putHTML(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink"));
userAgent = null;
if (seed.hash.equals(yacyCore.seedDB.mySeed().hash)) {
final HttpClient httpClient = new JakartaCommonsHttpClient(10000, null, null);
final JakartaCommonsHttpClient httpClient = new JakartaCommonsHttpClient(10000, null, null);
userAgent = httpClient.getUserAgent();
location = HttpClient.generateLocation();
} else {

@ -58,9 +58,8 @@ import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpdByteCountInputStream;
import de.anomic.index.indexURLReference;
import de.anomic.plasma.plasmaCrawlProfile;
@ -178,8 +177,8 @@ public class SitemapParser extends DefaultHandler {
*/
public void parse() {
// download document
HttpClient client = new JakartaCommonsHttpClient(5000, null, null);
HttpResponse res = null;
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, null, null);
JakartaCommonsHttpResponse res = null;
try {
res = client.GET(siteMapURL.toString());
if (res.getStatusCode() != 200) {

@ -43,6 +43,7 @@
package de.anomic.data;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@ -56,13 +57,13 @@ import java.util.ArrayList;
import java.util.Date;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpHeader;
import de.anomic.http.HttpResponse.Saver;
import de.anomic.plasma.plasmaCrawlRobotsTxt;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyURL;
@ -405,8 +406,8 @@ public final class robotsParser{
// setup http-client
//TODO: adding Traffic statistic for robots download?
HttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders, null);
HttpResponse res = null;
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeaders, null);
JakartaCommonsHttpResponse res = null;
try {
// sending the get request
res = client.GET(robotsURL.toString());
@ -433,7 +434,11 @@ public final class robotsParser{
// downloading the content
serverByteBuffer sbb = new serverByteBuffer();
Saver.writeContent(res, new BufferedOutputStream(sbb), null);
try {
serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(sbb));
} finally {
res.closeStream();
}
robotsTxt = sbb.getBytes();
downloadEnd = System.currentTimeMillis();

@ -23,11 +23,11 @@
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http;
import java.io.IOException;
import java.util.Date;
import java.util.Map;
import de.anomic.server.logging.serverLog;
@ -37,6 +37,7 @@ import de.anomic.server.logging.serverLog;
* some methods must be implemented (the "socket-layer")
*/
public abstract class HttpClient {
private static final String systemOST;
static {
// provide system information for client identification
@ -68,73 +69,6 @@ public abstract class HttpClient {
return systemOST;
}
/**
* "This method can be used for obtaining metainformation about the entity implied by the request without
* transferring the entity-body itself" [RFC 2616, sect. 9.4]
*
* @param uri
* @return response header
* @throws IOException if data cannot be read
*/
public abstract HttpResponse HEAD(String uri) throws IOException;
/**
* "The GET method means retrieve whatever information [...] is identified by the Request-URI" [RFC 2616, sect. 9.3]
*
* @param uri
* @return response body
* @throws IOException if data cannot be read
*/
public abstract HttpResponse GET(String uri) throws IOException;
/**
* "Providing a block of data [...] to a data-handling process;" [RFC 2616, sect. 9.5]
*
* @param uri
* @param nameDataPairs
* @return response body
* @throws IOException if data cannot be read
*/
public abstract HttpResponse POST(String uri, Map<String, ?> nameDataPairs) throws IOException;
/**
* "for use with a proxy that can dynamically switch to being a tunnel" [RFC 2616, sect. 9.9]
*
* @param host
* @param port
* @return
* @throws IOException
*/
public abstract HttpResponse CONNECT(String host, int port) throws IOException;
/**
* use proxyConfig to establish the connection
*
* @param proxyConfig
*/
public abstract void setProxy(httpRemoteProxyConfig proxyConfig);
/**
* sets the header for all coming requests
*
* @param header may be null
*/
public abstract void setHeader(httpHeader header);
/**
* sets the timeout in milliseconds
*
* @param timeout
*/
public abstract void setTimeout(int timeout);
/**
* gives the user-agent used by this http-client
*
* @return
*/
public abstract String getUserAgent();
/**
* for easy access
*
@ -213,7 +147,7 @@ public abstract class HttpClient {
*/
public static byte[] wget(final String uri, httpHeader header, final String vhost, int timeout) {
assert uri != null : "precondition violated: uri != null";
final HttpClient client = new JakartaCommonsHttpClient(timeout, null, null);
final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, null, null);
// set header
header = addHostHeader(header, vhost);
@ -221,7 +155,7 @@ public abstract class HttpClient {
// do the request
try {
final HttpResponse response = client.GET(uri);
final JakartaCommonsHttpResponse response = client.GET(uri);
return response.getData();
} catch (final IOException e) {
serverLog.logWarning("HTTPC", "wget(" + uri + ") failed: " + e.getMessage());
@ -265,9 +199,9 @@ public abstract class HttpClient {
* @return
*/
public static httpHeader whead(final String uri, final httpHeader header) {
final de.anomic.http.HttpClient client = new JakartaCommonsHttpClient(10000, header, null);
final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, header, null);
try {
final HttpResponse response = client.HEAD(uri);
final JakartaCommonsHttpResponse response = client.HEAD(uri);
return response.getResponseHeader();
} catch (final IOException e) {
serverLog.logWarning("HTTPC", "whead(" + uri + ") failed: " + e.getMessage());

@ -1,135 +0,0 @@
// HttpResponse.java
// (C) 2008 by Daniel Raap; danielr@users.berlios.de
// first published 2.4.2008 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2008-03-14 01:16:04 +0100 (Fr, 14 Mrz 2008) $
// $LastChangedRevision: 4558 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import de.anomic.server.serverFileUtils;
/**
* @author daniel
*
*/
public interface HttpResponse {
/**
* returns the header as {@link de.anomic.http.httpHeader}
* @return
*/
public abstract httpHeader getResponseHeader();
/**
* get the body of the response
* @return the data
* @throws IOException
*/
public abstract byte[] getData() throws IOException;
/**
* returns a stream to read the body of the response
*
* <strong>Important: When done call <code>closeStream()</code>!</strong>
* @return
* @throws IOException
*/
public abstract InputStream getDataAsStream() throws IOException;
/**
* closes the input stream
*/
public abstract void closeStream();
/**
* "Statuscode SPACE Statusmessage"
* @return the statusLine
*/
public abstract String getStatusLine();
/**
* the status code of this response
* @return
* @ensure result >= 100 && result <= 999 (is a 3-digit integer)
*/
public abstract int getStatusCode();
/**
* HTTP version of the response
* @return
*/
public abstract String getHttpVer();
/**
* save a response to some storage
* @author daniel
*
*/
public static class Saver {
/**
* copies the body of the response accordingly to hfos to the destination
* @param res
* @param hfos OutputStream (raw bytes) or Writer (decode to characters)
* @param byteStream additional OutputStream where data is stored
* @throws IOException
* @throws UnsupportedEncodingException
*/
public static void writeContent(HttpResponse res, BufferedOutputStream hfos, BufferedOutputStream byteStream) throws IOException, UnsupportedEncodingException {
try {
InputStream data = res.getDataAsStream();
if (byteStream == null) {
serverFileUtils.copyToStream(new BufferedInputStream(data), hfos);
} else {
serverFileUtils.copyToStreams(new BufferedInputStream(data), hfos, byteStream);
}
} finally {
res.closeStream();
}
}
public static void writeContent(HttpResponse res, BufferedWriter hfos, OutputStream byteStream) throws IOException, UnsupportedEncodingException {
try {
InputStream data = res.getDataAsStream();
String charSet = httpHeader.getCharSet(res.getResponseHeader());
if (byteStream == null) {
serverFileUtils.copyToWriter(new BufferedInputStream(data), hfos, charSet);
} else {
serverFileUtils.copyToWriters(new BufferedInputStream(data), hfos, new BufferedWriter(new OutputStreamWriter(byteStream, charSet)) , charSet);
}
} finally {
res.closeStream();
}
}
}
}

@ -69,7 +69,7 @@ import de.anomic.yacy.yacyVersion;
* @author danielr
*
*/
public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
public class JakartaCommonsHttpClient {
/**
* "the HttpClient instance and connection manager should be shared among all threads for maximum efficiency."
* (Concurrent execution of HTTP methods, http://hc.apache.org/httpclient-3.x/performance.html)
@ -95,7 +95,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
* set options for connection manager
*/
// conManager.getParams().setDefaultMaxConnectionsPerHost(4); // default 2
conManager.getParams().setMaxTotalConnections(50); // default 20
conManager.getParams().setMaxTotalConnections(200); // Proxy may need many connections
conManager.getParams().setConnectionTimeout(60000); // set a default timeout
conManager.getParams().setDefaultMaxConnectionsPerHost(20); // prevent DoS by mistake
// TODO should this be configurable?
@ -184,7 +184,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
* @return InputStream of content (body)
* @throws IOException
*/
public HttpResponse GET(final String uri) throws IOException {
public JakartaCommonsHttpResponse GET(final String uri) throws IOException {
final HttpMethod get = new GetMethod(uri);
return execute(get);
}
@ -197,7 +197,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
* @return Instance of response with the content.
* @throws IOException
*/
public HttpResponse HEAD(final String uri) throws IOException {
public JakartaCommonsHttpResponse HEAD(final String uri) throws IOException {
assert uri != null : "precondition violated: uri != null";
final HttpMethod head = new HeadMethod(uri);
return execute(head);
@ -213,7 +213,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
* @return Instance of response with the content.
* @throws IOException
*/
public HttpResponse POST(final String uri, final InputStream ins) throws IOException {
public JakartaCommonsHttpResponse POST(final String uri, final InputStream ins) throws IOException {
assert uri != null : "precondition violated: uri != null";
assert ins != null : "precondition violated: ins != null";
final PostMethod post = new PostMethod(uri);
@ -231,7 +231,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
* @return Instance of response with the content.
* @throws IOException
*/
public HttpResponse POST(final String uri, final Map<String, ?> files) throws IOException {
public JakartaCommonsHttpResponse POST(final String uri, final Map<String, ?> files) throws IOException {
assert uri != null : "precondition violated: uri != null";
final PostMethod post = new PostMethod(uri);
@ -278,7 +278,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
*
* @see de.anomic.http.HttpClient#CONNECT(java.lang.String, int, de.anomic.http.httpHeader)
*/
public HttpResponse CONNECT(final String host, final int port) throws IOException {
public JakartaCommonsHttpResponse CONNECT(final String host, final int port) throws IOException {
final HostConfiguration hostConfig = new HostConfiguration();
hostConfig.setHost(host, port);
final HttpMethod connect = new ConnectMethod(hostConfig);
@ -369,7 +369,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
* @throws IOException
* @throws HttpException
*/
private HttpResponse execute(final HttpMethod method) throws IOException, HttpException {
private JakartaCommonsHttpResponse execute(final HttpMethod method) throws IOException, HttpException {
assert method != null : "precondition violated: method != null";
// set header
for (final Header header : headers) {
@ -480,7 +480,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
* @param args
*/
public static void main(final String[] args) {
HttpResponse resp = null;
JakartaCommonsHttpResponse resp = null;
String url = args[0];
if (!(url.toUpperCase().startsWith("HTTP://"))) {
url = "http://" + url;
@ -492,7 +492,7 @@ public class JakartaCommonsHttpClient extends de.anomic.http.HttpClient {
files.put("myfile.txt", "this is not a file ;)".getBytes());
files.put("anotherfile.raw", "this is not a binary file ;)".getBytes());
System.out.println("POST " + files.size() + " elements to " + url);
final de.anomic.http.HttpClient client = new JakartaCommonsHttpClient(1000, null, null);
final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(1000, null, null);
resp = client.POST(url, files);
System.out.println("----- Header: -----");
System.out.println(new String(resp.getResponseHeader().toString()));

@ -23,6 +23,7 @@
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.http;
import java.io.ByteArrayOutputStream;
@ -39,7 +40,7 @@ import org.apache.commons.httpclient.HttpMethod;
* @author daniel
* @since 21.03.2008
*/
public class JakartaCommonsHttpResponse implements HttpResponse {
public class JakartaCommonsHttpResponse {
private final HttpMethod method;
private String incomingAccountingName = null;

@ -62,7 +62,7 @@
package de.anomic.http;
import java.io.BufferedOutputStream;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
@ -75,6 +75,7 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.BindException;
import java.net.ConnectException;
@ -98,7 +99,6 @@ import java.util.zip.GZIPOutputStream;
import de.anomic.htmlFilter.htmlFilterContentTransformer;
import de.anomic.htmlFilter.htmlFilterTransformer;
import de.anomic.htmlFilter.htmlFilterWriter;
import de.anomic.http.HttpResponse.Saver;
import de.anomic.index.indexReferenceBlacklist;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaParser;
@ -144,7 +144,7 @@ public final class httpdProxyHandler {
* Do logging configuration for special proxy access log file
*/
static {
// Doing logger initialisation
// Doing logger initialization
try {
serverLog.logInfo("PROXY","Configuring proxy access logging ...");
@ -395,7 +395,7 @@ public final class httpdProxyHandler {
addXForwardedForHeader(conProp, requestHeader);
// decide wether to use a cache entry or connect to the network
// decide whether to use a cache entry or connect to the network
File cacheFile = plasmaHTCache.getCachePath(url);
httpHeader cachedResponseHeader = null;
@ -411,8 +411,8 @@ public final class httpdProxyHandler {
// why are files unzipped upon arrival? why not zip all files in cache?
// This follows from the following premises
// (a) no file shall be unzip-ed more than once to prevent unnessesary computing time
// (b) old cache entries shall be comparable with refill-entries to detect/distiguish case 3+4
// (a) no file shall be unzip-ed more than once to prevent unnecessary computing time
// (b) old cache entries shall be comparable with refill-entries to detect/distinguish case 3+4
// (c) the indexing mechanism needs files unzip-ed, a schedule could do that later
// case b and c contradicts, if we use a scheduler, because files in a stale cache would be unzipped
// and the newly arrival would be zipped and would have to be unzipped upon load. But then the
@ -487,7 +487,7 @@ public final class httpdProxyHandler {
httpChunkedOutputStream chunkedOut = null;
Writer hfos = null;
HttpResponse res = null;
JakartaCommonsHttpResponse res = null;
try {
String host = conProp.getProperty(httpHeader.CONNECTION_PROP_HOST);
@ -519,7 +519,7 @@ public final class httpdProxyHandler {
prepareRequestHeader(requestHeader, httpVer);
// setup HTTP-client
final HttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
final JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
final String connectHost = hostPart(host, port, yAddress);
final String getUrl = "http://"+ connectHost + remotePath;
@ -635,7 +635,7 @@ public final class httpdProxyHandler {
{
// ok, we don't write actually into a file, only to RAM, and schedule writing the file.
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
Saver.writeContent(res, new BufferedWriter(hfos), byteStream);
writeContent(res, new BufferedWriter(hfos), byteStream);
// cached bytes
byte[] cacheArray;
if(byteStream.size() > 0) {
@ -670,7 +670,7 @@ public final class httpdProxyHandler {
// the file is too big to cache it in the ram, or the size is unknown
// write to file right here.
cacheFile.getParentFile().mkdirs();
Saver.writeContent(res, new BufferedWriter(hfos), new FileOutputStream(cacheFile));
writeContent(res, new BufferedWriter(hfos), new FileOutputStream(cacheFile));
if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize();
theLogger.logFine("for write-file of " + url + ": contentLength = " + contentLength + ", sizeBeforeDelete = " + sizeBeforeDelete);
plasmaHTCache.writeFileAnnouncement(cacheFile);
@ -700,7 +700,7 @@ public final class httpdProxyHandler {
" StoreHTCache=" + storeHTCache +
" SupportetContent=" + isSupportedContent);
Saver.writeContent(res, new BufferedWriter(hfos), null);
writeContent(res, new BufferedWriter(hfos));
if (hfos instanceof htmlFilterWriter) ((htmlFilterWriter) hfos).finalize();
if (sizeBeforeDelete == -1) {
// no old file and no load. just data passing
@ -834,6 +834,28 @@ public final class httpdProxyHandler {
return;
}
public static void writeContent(JakartaCommonsHttpResponse res, BufferedWriter hfos) throws IOException, UnsupportedEncodingException {
try {
InputStream data = res.getDataAsStream();
if (data == null) return;
String charSet = httpHeader.getCharSet(res.getResponseHeader());
serverFileUtils.copyToWriter(new BufferedInputStream(data), hfos, charSet);
} finally {
res.closeStream();
}
}
public static void writeContent(JakartaCommonsHttpResponse res, BufferedWriter hfos, OutputStream byteStream) throws IOException, UnsupportedEncodingException {
assert byteStream != null;
try {
InputStream data = res.getDataAsStream();
if (data == null) return;
String charSet = httpHeader.getCharSet(res.getResponseHeader());
serverFileUtils.copyToWriters(new BufferedInputStream(data), hfos, new BufferedWriter(new OutputStreamWriter(byteStream, charSet)) , charSet);
} finally {
res.closeStream();
}
}
private static void removeHopByHopHeaders(httpHeader headers) {
/*
@ -868,7 +890,7 @@ public final class httpdProxyHandler {
public static void doHead(Properties conProp, httpHeader requestHeader, OutputStream respond) {
HttpResponse res = null;
JakartaCommonsHttpResponse res = null;
yacyURL url = null;
try {
// remembering the starting time of the request
@ -935,7 +957,7 @@ public final class httpdProxyHandler {
prepareRequestHeader(requestHeader, httpVer);
// setup HTTP-client
HttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, requestHeader, null);
// generate request-url
final String connectHost = hostPart(host, port, yAddress);
@ -1057,7 +1079,7 @@ public final class httpdProxyHandler {
serverFileUtils.copy(body, buffer, requestLength);
body = new ByteArrayInputStream(buffer.toByteArray());
}
HttpResponse res = null;
JakartaCommonsHttpResponse res = null;
try {
// sending the request
res = client.POST(getUrl, body);
@ -1104,8 +1126,17 @@ public final class httpdProxyHandler {
responseHeader);
// respondHeader(respond, res.status, res.responseHeader);
Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond), null);
// Saver.writeContent(res, (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond));
/*
// *** (†bernommen aus Saver-Klasse: warum ist dies hier die einzige Methode, die einen OutputStream statt einen Writer benutzt?)
try {
serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), (chunked != null) ? new BufferedOutputStream(chunked) : new BufferedOutputStream(respond));
} finally {
res.closeStream();
}
if (chunked != null) chunked.finish();
*/
writeContent(res, new BufferedWriter(new OutputStreamWriter((chunked != null) ? chunked : respond)));
respond.flush();
} finally {
@ -1208,9 +1239,9 @@ public final class httpdProxyHandler {
(proxyConfig.useProxy()) &&
(proxyConfig.useProxy4SSL())
) {
HttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig);
JakartaCommonsHttpClient remoteProxy = new JakartaCommonsHttpClient(timeout, requestHeader, proxyConfig);
HttpResponse response = null;
JakartaCommonsHttpResponse response = null;
try {
response = remoteProxy.CONNECT(host, port);
// outputs a logline to the serverlog with the current status

@ -36,9 +36,8 @@ import java.util.HashSet;
import java.util.Iterator;
import de.anomic.data.htmlTools;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCache;
@ -257,9 +256,9 @@ public final class indexRepositoryReference {
yacyURL newUrl = new yacyURL(newUrlStr, null);
// doing a http head request to test if the url is correct
HttpClient client = new JakartaCommonsHttpClient(10000, null, null);
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, null, null);
client.setProxy(proxyConfig);
HttpResponse res = null;
JakartaCommonsHttpResponse res = null;
try {
res = client.HEAD(newUrl.toString());
} finally {

@ -55,7 +55,6 @@ import java.net.UnknownHostException;
import java.util.Date;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpHeader;
@ -177,9 +176,9 @@ public final class plasmaHTTPLoader {
requestHeader.put(httpHeader.ACCEPT_ENCODING, this.acceptEncoding);
// HTTP-Client
HttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader, null);
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(socketTimeout, requestHeader, null);
HttpResponse res = null;
JakartaCommonsHttpResponse res = null;
try {
// send request
res = client.GET(entry.url().toString());

@ -54,9 +54,8 @@ import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.htmlFilter.htmlFilterImageEntry;
import de.anomic.htmlFilter.htmlFilterInputStream;
import de.anomic.htmlFilter.htmlFilterWriter;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.plasma.parser.Parser;
import de.anomic.plasma.parser.ParserException;
import de.anomic.plasma.parser.ParserInfo;
@ -847,7 +846,7 @@ public final class plasmaParser {
}
String mode = args[0];
HttpResponse res = null;
JakartaCommonsHttpResponse res = null;
plasmaParserDocument document = null;
try { // close InputStream when done
if (mode.equalsIgnoreCase("-f")) {
@ -857,7 +856,7 @@ public final class plasmaParser {
contentURL = new yacyURL(args[1], null);
// downloading the document content
HttpClient client = new JakartaCommonsHttpClient(5000, null, null);
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(5000, null, null);
res = client.GET(args[1]);
if (res.getStatusCode() != 200) {

@ -53,8 +53,8 @@ import java.util.Map;
import java.util.TreeMap;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpHeader;
import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpdProxyHandler;
@ -257,7 +257,7 @@ public final class yacyClient {
* @throws IOException
*/
private static byte[] wput(final String url, String vhost, final Map<String, ?> post, final int timeout) throws IOException {
HttpClient client = new JakartaCommonsHttpClient(timeout, null, null);
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(timeout, null, null);
client.setProxy(proxyConfig());
// address vhost
@ -265,7 +265,7 @@ public final class yacyClient {
header.add(httpHeader.HOST, vhost);
client.setHeader(header);
HttpResponse res = null;
JakartaCommonsHttpResponse res = null;
byte[] content = null;
try {
// send request/data

@ -60,9 +60,8 @@ import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpHeader;
import de.anomic.http.httpd;
import de.anomic.kelondro.kelondroBase64Order;
@ -817,9 +816,9 @@ public final class yacySeedDB {
reqHeader.put(httpHeader.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
// init http-client
HttpClient client = new JakartaCommonsHttpClient(10000, reqHeader, null);
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(10000, reqHeader, null);
byte[] content = null;
HttpResponse res = null;
JakartaCommonsHttpResponse res = null;
try {
// send request
res = client.GET(seedURL.toString());

@ -27,6 +27,7 @@
package de.anomic.yacy;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
@ -41,12 +42,11 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import de.anomic.htmlFilter.htmlFilterContentScraper;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.http.HttpResponse.Saver;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverSystem;
import de.anomic.server.logging.serverLog;
@ -330,11 +330,15 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
File storagePath = plasmaSwitchboard.getSwitchboard().releasePath;
// load file
File download = new File(storagePath, release.url.getFileName());
HttpClient client = new JakartaCommonsHttpClient(120000, null, null);
HttpResponse res = null;
JakartaCommonsHttpClient client = new JakartaCommonsHttpClient(120000, null, null);
JakartaCommonsHttpResponse res = null;
try {
res = client.GET(release.url.toString());
Saver.writeContent(res, new BufferedOutputStream(new FileOutputStream(download)), null);
try {
serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(new FileOutputStream(download)));
} finally {
res.closeStream();
}
if ((!download.exists()) || (download.length() == 0)) throw new IOException("wget of url " + release.url + " failed");
} catch (IOException e) {
serverLog.logSevere("yacyVersion", "download of " + release.name + " failed: " + e.getMessage());

@ -41,6 +41,7 @@
// the intact and unchanged copyright notice.
// Contributions and changes to the program code must be marked as such.
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@ -67,10 +68,9 @@ import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import de.anomic.data.translator;
import de.anomic.http.HttpClient;
import de.anomic.http.HttpResponse;
import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpHeader;
import de.anomic.http.httpd;
import de.anomic.http.HttpResponse.Saver;
import de.anomic.http.JakartaCommonsHttpClient;
import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry;
@ -529,8 +529,8 @@ public final class yacy {
// send 'wget' to web interface
httpHeader requestHeader = new httpHeader();
requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify
HttpClient con = new JakartaCommonsHttpClient(10000, requestHeader, null);
HttpResponse res = null;
JakartaCommonsHttpClient con = new JakartaCommonsHttpClient(10000, requestHeader, null);
JakartaCommonsHttpResponse res = null;
try {
res = con.GET("http://localhost:"+ port +"/Steering.html?shutdown=");
@ -539,7 +539,11 @@ public final class yacy {
serverLog.logConfig("REMOTE-SHUTDOWN", "YACY accepted shutdown command.");
serverLog.logConfig("REMOTE-SHUTDOWN", "Stand by for termination, which may last some seconds.");
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Saver.writeContent(res, new BufferedOutputStream(bos), null);
try {
serverFileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(bos));
} finally {
res.closeStream();
}
} else {
serverLog.logSevere("REMOTE-SHUTDOWN", "error response from YACY socket: " + res.getStatusLine());
System.exit(-1);

Loading…
Cancel
Save